< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page

   1 /*
   2  * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.

   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "memory/allocation.hpp"
  28 #include "memory/universe.hpp"
  29 
  30 #include "gc/shared/classUnloadingContext.hpp"
  31 #include "gc/shared/gcArguments.hpp"
  32 #include "gc/shared/gcTimer.hpp"
  33 #include "gc/shared/gcTraceTime.inline.hpp"
  34 #include "gc/shared/locationPrinter.inline.hpp"
  35 #include "gc/shared/memAllocator.hpp"
  36 #include "gc/shared/plab.hpp"
  37 #include "gc/shared/tlab_globals.hpp"
  38 




  39 #include "gc/shenandoah/shenandoahBarrierSet.hpp"

  40 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  41 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  42 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  43 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  45 #include "gc/shenandoah/shenandoahControlThread.hpp"
  46 #include "gc/shenandoah/shenandoahFreeSet.hpp"



  47 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  48 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  49 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  50 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  51 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  52 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  53 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  54 #include "gc/shenandoah/shenandoahMetrics.hpp"
  55 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"

  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"

  62 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  63 #include "gc/shenandoah/shenandoahUtils.hpp"
  64 #include "gc/shenandoah/shenandoahVerifier.hpp"
  65 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  66 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  67 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  68 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"


  69 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  70 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  71 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"


  72 #if INCLUDE_JFR
  73 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  74 #endif
  75 
  76 #include "classfile/systemDictionary.hpp"
  77 #include "code/codeCache.hpp"
  78 #include "memory/classLoaderMetaspace.hpp"
  79 #include "memory/metaspaceUtils.hpp"
  80 #include "oops/compressedOops.inline.hpp"
  81 #include "prims/jvmtiTagMap.hpp"
  82 #include "runtime/atomic.hpp"
  83 #include "runtime/globals.hpp"
  84 #include "runtime/interfaceSupport.inline.hpp"
  85 #include "runtime/java.hpp"
  86 #include "runtime/orderAccess.hpp"
  87 #include "runtime/safepointMechanism.hpp"
  88 #include "runtime/vmThread.hpp"
  89 #include "services/mallocTracker.hpp"
  90 #include "services/memTracker.hpp"
  91 #include "utilities/events.hpp"

 143 jint ShenandoahHeap::initialize() {
 144   //
 145   // Figure out heap sizing
 146   //
 147 
 148   size_t init_byte_size = InitialHeapSize;
 149   size_t min_byte_size  = MinHeapSize;
 150   size_t max_byte_size  = MaxHeapSize;
 151   size_t heap_alignment = HeapAlignment;
 152 
 153   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 154 
 155   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 156   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 157 
 158   _num_regions = ShenandoahHeapRegion::region_count();
 159   assert(_num_regions == (max_byte_size / reg_size_bytes),
 160          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 161          _num_regions, max_byte_size, reg_size_bytes);
 162 
 163   // Now we know the number of regions, initialize the heuristics.
 164   initialize_heuristics();
 165 
 166   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 167   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 168   assert(num_committed_regions <= _num_regions, "sanity");
 169   _initial_size = num_committed_regions * reg_size_bytes;
 170 
 171   size_t num_min_regions = min_byte_size / reg_size_bytes;
 172   num_min_regions = MIN2(num_min_regions, _num_regions);
 173   assert(num_min_regions <= _num_regions, "sanity");
 174   _minimum_size = num_min_regions * reg_size_bytes;
 175 
 176   // Default to max heap size.
 177   _soft_max_size = _num_regions * reg_size_bytes;
 178 
 179   _committed = _initial_size;
 180 



 181   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 182   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 183   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 184 
 185   //
 186   // Reserve and commit memory for heap
 187   //
 188 
 189   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 190   initialize_reserved_region(heap_rs);
 191   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 192   _heap_region_special = heap_rs.special();
 193 
 194   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 195          "Misaligned heap: " PTR_FORMAT, p2i(base()));



 196 
 197 #if SHENANDOAH_OPTIMIZED_MARKTASK
 198   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 199   // Fail if we ever attempt to address more than we can.
 200   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 201     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 202                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 203                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 204                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 205     vm_exit_during_initialization("Fatal Error", buf);
 206   }
 207 #endif
 208 
 209   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 210   if (!_heap_region_special) {
 211     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 212                               "Cannot commit heap memory");
 213   }
 214 




























 215   //
 216   // Reserve and commit memory for bitmap(s)
 217   //
 218 
 219   _bitmap_size = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 220   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 221 
 222   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 223 
 224   guarantee(bitmap_bytes_per_region != 0,
 225             "Bitmap bytes per region should not be zero");
 226   guarantee(is_power_of_2(bitmap_bytes_per_region),
 227             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 228 
 229   if (bitmap_page_size > bitmap_bytes_per_region) {
 230     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 231     _bitmap_bytes_per_slice = bitmap_page_size;
 232   } else {
 233     _bitmap_regions_per_slice = 1;
 234     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 235   }
 236 
 237   guarantee(_bitmap_regions_per_slice >= 1,
 238             "Should have at least one region per slice: " SIZE_FORMAT,
 239             _bitmap_regions_per_slice);
 240 
 241   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 242             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 243             _bitmap_bytes_per_slice, bitmap_page_size);
 244 
 245   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);




 246   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 247   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 248   _bitmap_region_special = bitmap.special();
 249 
 250   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 251                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 252   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 253   if (!_bitmap_region_special) {
 254     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 255                               "Cannot commit bitmap memory");
 256   }
 257 
 258   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
 259 
 260   if (ShenandoahVerify) {
 261     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);




 262     if (!verify_bitmap.special()) {
 263       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 264                                 "Cannot commit verification bitmap memory");
 265     }
 266     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 267     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 268     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 269     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 270   }
 271 
 272   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 273   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);












 274   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 275   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 276   _aux_bitmap_region_special = aux_bitmap.special();
 277   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 278 
 279   //
 280   // Create regions and region sets
 281   //
 282   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 283   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 284   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());

 285 
 286   ReservedSpace region_storage(region_storage_size, region_page_size);



 287   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 288   if (!region_storage.special()) {
 289     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 290                               "Cannot commit region memory");
 291   }
 292 
 293   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 294   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 295   // If not successful, bite a bullet and allocate at whatever address.
 296   {
 297     size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 298     size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);

 299 
 300     uintptr_t min = round_up_power_of_2(cset_align);
 301     uintptr_t max = (1u << 30u);

 302 
 303     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 304       char* req_addr = (char*)addr;
 305       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 306       ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size(), req_addr);
 307       if (cset_rs.is_reserved()) {
 308         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 309         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 310         break;
 311       }
 312     }
 313 
 314     if (_collection_set == nullptr) {
 315       ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size());
 316       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 317     }




 318   }
 319 
 320   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);

 321   _free_set = new ShenandoahFreeSet(this, _num_regions);
 322 
 323   {
 324     ShenandoahHeapLocker locker(lock());
 325 

 326     for (size_t i = 0; i < _num_regions; i++) {
 327       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 328       bool is_committed = i < num_committed_regions;
 329       void* loc = region_storage.base() + i * region_align;
 330 
 331       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 332       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 333 
 334       _marking_context->initialize_top_at_mark_start(r);
 335       _regions[i] = r;
 336       assert(!collection_set()->is_in(i), "New region should not be in collection set");


 337     }
 338 
 339     // Initialize to complete
 340     _marking_context->mark_complete();

 341 
 342     _free_set->rebuild();



 343   }
 344 
 345   if (AlwaysPreTouch) {
 346     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 347     // before initialize() below zeroes it with initializing thread. For any given region,
 348     // we touch the region and the corresponding bitmaps from the same thread.
 349     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 350 
 351     _pretouch_heap_page_size = heap_page_size;
 352     _pretouch_bitmap_page_size = bitmap_page_size;
 353 
 354 #ifdef LINUX
 355     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 356     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 357     // them into huge one. Therefore, we need to pretouch with smaller pages.
 358     if (UseTransparentHugePages) {
 359       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 360       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 361     }
 362 #endif

 379   for (uint worker = 0; worker < _max_workers; worker++) {
 380     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 381     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 382   }
 383 
 384   // There should probably be Shenandoah-specific options for these,
 385   // just as there are G1-specific options.
 386   {
 387     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 388     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 389     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 390   }
 391 
 392   _monitoring_support = new ShenandoahMonitoringSupport(this);
 393   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 394   ShenandoahCodeRoots::initialize();
 395 
 396   if (ShenandoahPacing) {
 397     _pacer = new ShenandoahPacer(this);
 398     _pacer->setup_for_idle();
 399   } else {
 400     _pacer = nullptr;
 401   }
 402 
 403   _control_thread = new ShenandoahControlThread();
 404 
 405   ShenandoahInitLogger::print();
 406 
 407   return JNI_OK;
 408 }
 409 
 410 void ShenandoahHeap::initialize_mode() {








 411   if (ShenandoahGCMode != nullptr) {
 412     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 413       _gc_mode = new ShenandoahSATBMode();
 414     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 415       _gc_mode = new ShenandoahIUMode();
 416     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 417       _gc_mode = new ShenandoahPassiveMode();


 418     } else {
 419       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 420     }
 421   } else {
 422     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 423   }
 424   _gc_mode->initialize_flags();
 425   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 426     vm_exit_during_initialization(
 427             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 428                     _gc_mode->name()));
 429   }
 430   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 431     vm_exit_during_initialization(
 432             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 433                     _gc_mode->name()));
 434   }
 435 }
 436 
 437 void ShenandoahHeap::initialize_heuristics() {
 438   assert(_gc_mode != nullptr, "Must be initialized");
 439   _heuristics = _gc_mode->initialize_heuristics();






 440 
 441   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 442     vm_exit_during_initialization(
 443             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 444                     _heuristics->name()));
 445   }
 446   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 447     vm_exit_during_initialization(
 448             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 449                     _heuristics->name()));
 450   }

 451 }
 452 
 453 #ifdef _MSC_VER
 454 #pragma warning( push )
 455 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 456 #endif
 457 
 458 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 459   CollectedHeap(),

 460   _initial_size(0),
 461   _used(0),
 462   _committed(0),
 463   _bytes_allocated_since_gc_start(0),
 464   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 465   _workers(nullptr),
 466   _safepoint_workers(nullptr),
 467   _heap_region_special(false),
 468   _num_regions(0),
 469   _regions(nullptr),
 470   _update_refs_iterator(this),
 471   _gc_state_changed(false),







 472   _control_thread(nullptr),
 473   _shenandoah_policy(policy),
 474   _gc_mode(nullptr),
 475   _heuristics(nullptr),
 476   _free_set(nullptr),
 477   _pacer(nullptr),
 478   _verifier(nullptr),
 479   _phase_timings(nullptr),



 480   _monitoring_support(nullptr),
 481   _memory_pool(nullptr),
 482   _stw_memory_manager("Shenandoah Pauses"),
 483   _cycle_memory_manager("Shenandoah Cycles"),
 484   _gc_timer(new ConcurrentGCTimer()),
 485   _soft_ref_policy(),
 486   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 487   _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
 488   _marking_context(nullptr),
 489   _bitmap_size(0),
 490   _bitmap_regions_per_slice(0),
 491   _bitmap_bytes_per_slice(0),
 492   _bitmap_region_special(false),
 493   _aux_bitmap_region_special(false),
 494   _liveness_cache(nullptr),
 495   _collection_set(nullptr)

 496 {
 497   // Initialize GC mode early, so we can adjust barrier support
 498   initialize_mode();
 499   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 500 
 501   _max_workers = MAX2(_max_workers, 1U);
 502   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 503   if (_workers == nullptr) {
 504     vm_exit_during_initialization("Failed necessary allocation.");
 505   } else {
 506     _workers->initialize_workers();
 507   }
 508 
 509   if (ParallelGCThreads > 1) {
 510     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
 511                                                 ParallelGCThreads);
 512     _safepoint_workers->initialize_workers();
 513   }
 514 }
 515 
 516 #ifdef _MSC_VER
 517 #pragma warning( pop )
 518 #endif
 519 
 520 class ShenandoahResetBitmapTask : public WorkerTask {
 521 private:
 522   ShenandoahRegionIterator _regions;
 523 
 524 public:
 525   ShenandoahResetBitmapTask() :
 526     WorkerTask("Shenandoah Reset Bitmap") {}
 527 
 528   void work(uint worker_id) {
 529     ShenandoahHeapRegion* region = _regions.next();
 530     ShenandoahHeap* heap = ShenandoahHeap::heap();
 531     ShenandoahMarkingContext* const ctx = heap->marking_context();
 532     while (region != nullptr) {
 533       if (heap->is_bitmap_slice_committed(region)) {
 534         ctx->clear_bitmap(region);
 535       }
 536       region = _regions.next();
 537     }
 538   }
 539 };
 540 
 541 void ShenandoahHeap::reset_mark_bitmap() {
 542   assert_gc_workers(_workers->active_workers());
 543   mark_incomplete_marking_context();
 544 
 545   ShenandoahResetBitmapTask task;
 546   _workers->run_task(&task);
 547 }
 548 
 549 void ShenandoahHeap::print_on(outputStream* st) const {
 550   st->print_cr("Shenandoah Heap");
 551   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 552                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 553                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 554                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 555                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 556   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 557                num_regions(),
 558                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 559                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 560 
 561   st->print("Status: ");
 562   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 563   if (is_concurrent_mark_in_progress())        st->print("marking, ");

 564   if (is_evacuation_in_progress())             st->print("evacuating, ");
 565   if (is_update_refs_in_progress())            st->print("updating refs, ");
 566   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 567   if (is_full_gc_in_progress())                st->print("full gc, ");
 568   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 569   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 570   if (is_concurrent_strong_root_in_progress() &&
 571       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 572 
 573   if (cancelled_gc()) {
 574     st->print("cancelled");
 575   } else {
 576     st->print("not cancelled");
 577   }
 578   st->cr();
 579 
 580   st->print_cr("Reserved region:");
 581   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 582                p2i(reserved_region().start()),
 583                p2i(reserved_region().end()));

 594   st->cr();
 595   MetaspaceUtils::print_on(st);
 596 
 597   if (Verbose) {
 598     st->cr();
 599     print_heap_regions_on(st);
 600   }
 601 }
 602 
 603 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 604 public:
 605   void do_thread(Thread* thread) {
 606     assert(thread != nullptr, "Sanity");
 607     assert(thread->is_Worker_thread(), "Only worker thread expected");
 608     ShenandoahThreadLocalData::initialize_gclab(thread);
 609   }
 610 };
 611 
 612 void ShenandoahHeap::post_initialize() {
 613   CollectedHeap::post_initialize();


 614   MutexLocker ml(Threads_lock);
 615 
 616   ShenandoahInitWorkerGCLABClosure init_gclabs;
 617   _workers->threads_do(&init_gclabs);
 618 
 619   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 620   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 621   _workers->set_initialize_gclab();
 622   if (_safepoint_workers != nullptr) {
 623     _safepoint_workers->threads_do(&init_gclabs);
 624     _safepoint_workers->set_initialize_gclab();
 625   }
 626 
 627   _heuristics->initialize();
 628 
 629   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 630 }
 631 




 632 size_t ShenandoahHeap::used() const {
 633   return Atomic::load(&_used);
 634 }
 635 
 636 size_t ShenandoahHeap::committed() const {
 637   return Atomic::load(&_committed);
 638 }
 639 
 640 void ShenandoahHeap::increase_committed(size_t bytes) {
 641   shenandoah_assert_heaplocked_or_safepoint();
 642   _committed += bytes;
 643 }
 644 
 645 void ShenandoahHeap::decrease_committed(size_t bytes) {
 646   shenandoah_assert_heaplocked_or_safepoint();
 647   _committed -= bytes;
 648 }
 649 
 650 void ShenandoahHeap::increase_used(size_t bytes) {
 651   Atomic::add(&_used, bytes, memory_order_relaxed);









































 652 }
 653 
 654 void ShenandoahHeap::set_used(size_t bytes) {
 655   Atomic::store(&_used, bytes);



 656 }
 657 
 658 void ShenandoahHeap::decrease_used(size_t bytes) {
 659   assert(used() >= bytes, "never decrease heap size by more than we've left");
 660   Atomic::sub(&_used, bytes, memory_order_relaxed);


 661 }
 662 
 663 void ShenandoahHeap::increase_allocated(size_t bytes) {
 664   Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);



 665 }
 666 
 667 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 668   size_t bytes = words * HeapWordSize;
 669   if (!waste) {
 670     increase_used(bytes);
 671   }
 672   increase_allocated(bytes);


 673   if (ShenandoahPacing) {
 674     control_thread()->pacing_notify_alloc(words);
 675     if (waste) {
 676       pacer()->claim_for_alloc(words, true);
 677     }
 678   }
 679 }
 680 
 681 size_t ShenandoahHeap::capacity() const {
 682   return committed();
 683 }
 684 
 685 size_t ShenandoahHeap::max_capacity() const {
 686   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 687 }
 688 
 689 size_t ShenandoahHeap::soft_max_capacity() const {
 690   size_t v = Atomic::load(&_soft_max_size);
 691   assert(min_capacity() <= v && v <= max_capacity(),
 692          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 693          min_capacity(), v, max_capacity());
 694   return v;
 695 }
 696 
 697 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 698   assert(min_capacity() <= v && v <= max_capacity(),
 699          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 700          min_capacity(), v, max_capacity());
 701   Atomic::store(&_soft_max_size, v);
 702 }
 703 
 704 size_t ShenandoahHeap::min_capacity() const {
 705   return _minimum_size;
 706 }
 707 
 708 size_t ShenandoahHeap::initial_capacity() const {
 709   return _initial_size;
 710 }
 711 
 712 bool ShenandoahHeap::is_in(const void* p) const {
 713   HeapWord* heap_base = (HeapWord*) base();
 714   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 715   return p >= heap_base && p < last_region_end;





















 716 }
 717 
 718 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
 719   assert (ShenandoahUncommit, "should be enabled");
 720 
 721   // Application allocates from the beginning of the heap, and GC allocates at
 722   // the end of it. It is more efficient to uncommit from the end, so that applications
 723   // could enjoy the near committed regions. GC allocations are much less frequent,
 724   // and therefore can accept the committing costs.
 725 
 726   size_t count = 0;
 727   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 728     ShenandoahHeapRegion* r = get_region(i - 1);
 729     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 730       ShenandoahHeapLocker locker(lock());
 731       if (r->is_empty_committed()) {
 732         if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
 733           break;
 734         }
 735 
 736         r->make_uncommitted();
 737         count++;
 738       }
 739     }
 740     SpinPause(); // allow allocators to take the lock
 741   }
 742 
 743   if (count > 0) {
 744     control_thread()->notify_heap_changed();

















 745   }
















 746 }
 747 
 748 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 749   // New object should fit the GCLAB size
 750   size_t min_size = MAX2(size, PLAB::min_size());
 751 
 752   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 753   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;

 754   new_size = MIN2(new_size, PLAB::max_size());
 755   new_size = MAX2(new_size, PLAB::min_size());
 756 
 757   // Record new heuristic value even if we take any shortcut. This captures
 758   // the case when moderately-sized objects always take a shortcut. At some point,
 759   // heuristics should catch up with them.

 760   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 761 
 762   if (new_size < size) {
 763     // New size still does not fit the object. Fall back to shared allocation.
 764     // This avoids retiring perfectly good GCLABs, when we encounter a large object.

 765     return nullptr;
 766   }
 767 
 768   // Retire current GCLAB, and allocate a new one.
 769   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 770   gclab->retire();
 771 
 772   size_t actual_size = 0;
 773   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 774   if (gclab_buf == nullptr) {
 775     return nullptr;
 776   }
 777 
 778   assert (size <= actual_size, "allocation should fit");
 779 
 780   if (ZeroTLAB) {
 781     // ..and clear it.
 782     Copy::zero_to_words(gclab_buf, actual_size);
 783   } else {
 784     // ...and zap just allocated object.
 785 #ifdef ASSERT
 786     // Skip mangling the space corresponding to the object header to
 787     // ensure that the returned space is not considered parsable by
 788     // any concurrent GC thread.
 789     size_t hdr_size = oopDesc::header_size();
 790     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 791 #endif // ASSERT
 792   }
 793   gclab->set_buf(gclab_buf, actual_size);
 794   return gclab->allocate(size);
 795 }
 796 

 797 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 798                                             size_t requested_size,
 799                                             size_t* actual_size) {
 800   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 801   HeapWord* res = allocate_memory(req);
 802   if (res != nullptr) {
 803     *actual_size = req.actual_size();
 804   } else {
 805     *actual_size = 0;
 806   }
 807   return res;
 808 }
 809 
 810 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 811                                              size_t word_size,
 812                                              size_t* actual_size) {
 813   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 814   HeapWord* res = allocate_memory(req);
 815   if (res != nullptr) {
 816     *actual_size = req.actual_size();

 818     *actual_size = 0;
 819   }
 820   return res;
 821 }
 822 
 823 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 824   intptr_t pacer_epoch = 0;
 825   bool in_new_region = false;
 826   HeapWord* result = nullptr;
 827 
 828   if (req.is_mutator_alloc()) {
 829     if (ShenandoahPacing) {
 830       pacer()->pace_for_alloc(req.size());
 831       pacer_epoch = pacer()->epoch();
 832     }
 833 
 834     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 835       result = allocate_memory_under_lock(req, in_new_region);
 836     }
 837 
 838     // Allocation failed, block until control thread reacted, then retry allocation.












 839     //
 840     // It might happen that one of the threads requesting allocation would unblock
 841     // way later after GC happened, only to fail the second allocation, because
 842     // other threads have already depleted the free storage. In this case, a better
 843     // strategy is to try again, as long as GC makes progress (or until at least
 844     // one full GC has completed).
 845     size_t original_count = shenandoah_policy()->full_gc_count();
 846     while (result == nullptr
 847         && (_progress_last_gc.is_set() || original_count == shenandoah_policy()->full_gc_count())) {
 848       control_thread()->handle_alloc_failure(req);
 849       result = allocate_memory_under_lock(req, in_new_region);
 850     }






 851   } else {
 852     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 853     result = allocate_memory_under_lock(req, in_new_region);
 854     // Do not call handle_alloc_failure() here, because we cannot block.
 855     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 856   }
 857 
 858   if (in_new_region) {
 859     control_thread()->notify_heap_changed();
 860   }
 861 








 862   if (result != nullptr) {
 863     size_t requested = req.size();
 864     size_t actual = req.actual_size();
 865 
 866     assert (req.is_lab_alloc() || (requested == actual),
 867             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 868             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 869 
 870     if (req.is_mutator_alloc()) {
 871       notify_mutator_alloc_words(actual, false);
 872 
 873       // If we requested more than we were granted, give the rest back to pacer.
 874       // This only matters if we are in the same pacing epoch: do not try to unpace
 875       // over the budget for the other phase.
 876       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 877         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 878       }
 879     } else {
 880       increase_used(actual*HeapWordSize);
 881     }
 882   }
 883 
 884   return result;
 885 }
 886 
 887 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 888   ShenandoahHeapLocker locker(lock());
 889   return _free_set->allocate(req, in_new_region);








































 890 }
 891 
 892 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 893                                         bool*  gc_overhead_limit_was_exceeded) {
 894   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 895   return allocate_memory(req);
 896 }
 897 
 898 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 899                                                              size_t size,
 900                                                              Metaspace::MetadataType mdtype) {
 901   MetaWord* result;
 902 
 903   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 904   if (heuristics()->can_unload_classes()) {
 905     ShenandoahHeuristics* h = heuristics();
 906     h->record_metaspace_oom();
 907   }
 908 
 909   // Expand and retry allocation
 910   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 911   if (result != nullptr) {
 912     return result;
 913   }
 914 
 915   // Start full GC
 916   collect(GCCause::_metadata_GC_clear_soft_refs);
 917 
 918   // Retry allocation
 919   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 920   if (result != nullptr) {
 921     return result;
 922   }
 923 
 924   // Expand and retry allocation
 925   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

 978 private:
 979   void do_work() {
 980     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 981     ShenandoahHeapRegion* r;
 982     while ((r =_cs->claim_next()) != nullptr) {
 983       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 984       _sh->marked_object_iterate(r, &cl);
 985 
 986       if (ShenandoahPacing) {
 987         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 988       }
 989 
 990       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 991         break;
 992       }
 993     }
 994   }
 995 };
 996 
 997 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
 998   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
 999   workers()->run_task(&task);











































































































1000 }
1001 
1002 void ShenandoahHeap::trash_cset_regions() {
1003   ShenandoahHeapLocker locker(lock());
1004 
1005   ShenandoahCollectionSet* set = collection_set();
1006   ShenandoahHeapRegion* r;
1007   set->clear_current_index();
1008   while ((r = set->next()) != nullptr) {
1009     r->make_trash();
1010   }
1011   collection_set()->clear();
1012 }
1013 
1014 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1015   st->print_cr("Heap Regions:");
1016   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1017   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1018   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1019   st->print_cr("UWM=update watermark, U=used");
1020   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1021   st->print_cr("S=shared allocs, L=live data");
1022   st->print_cr("CP=critical pins");
1023 
1024   for (size_t i = 0; i < num_regions(); i++) {
1025     get_region(i)->print_on(st);
1026   }
1027 }
1028 
1029 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1030   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1031 
1032   oop humongous_obj = cast_to_oop(start->bottom());
1033   size_t size = humongous_obj->size();
1034   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1035   size_t index = start->index() + required_regions - 1;
1036 
1037   assert(!start->has_live(), "liveness must be zero");
1038 
1039   for(size_t i = 0; i < required_regions; i++) {
1040     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1041     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1042     ShenandoahHeapRegion* region = get_region(index --);
1043 
1044     assert(region->is_humongous(), "expect correct humongous start or continuation");
1045     assert(!region->is_cset(), "Humongous region should not be in collection set");
1046 
1047     region->make_trash_immediate();
1048   }

1049 }
1050 
1051 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1052 public:
1053   ShenandoahCheckCleanGCLABClosure() {}
1054   void do_thread(Thread* thread) {
1055     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1056     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1057     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");






1058   }
1059 };
1060 
1061 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1062 private:
1063   bool const _resize;
1064 public:
1065   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1066   void do_thread(Thread* thread) {
1067     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1068     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1069     gclab->retire();
1070     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1071       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1072     }













1073   }
1074 };
1075 
1076 void ShenandoahHeap::labs_make_parsable() {
1077   assert(UseTLAB, "Only call with UseTLAB");
1078 
1079   ShenandoahRetireGCLABClosure cl(false);
1080 
1081   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1082     ThreadLocalAllocBuffer& tlab = t->tlab();
1083     tlab.make_parsable();
1084     cl.do_thread(t);
1085   }
1086 
1087   workers()->threads_do(&cl);
1088 }
1089 
1090 void ShenandoahHeap::tlabs_retire(bool resize) {
1091   assert(UseTLAB, "Only call with UseTLAB");
1092   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1154   }
1155   return nullptr;
1156 }
1157 
1158 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1159   ShenandoahHeapRegion* r = heap_region_containing(addr);
1160   return r->block_is_obj(addr);
1161 }
1162 
1163 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1164   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1165 }
1166 
1167 void ShenandoahHeap::prepare_for_verify() {
1168   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1169     labs_make_parsable();
1170   }
1171 }
1172 
1173 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1174   tcl->do_thread(_control_thread);







1175   workers()->threads_do(tcl);
1176   if (_safepoint_workers != nullptr) {
1177     _safepoint_workers->threads_do(tcl);
1178   }
1179 }
1180 
1181 void ShenandoahHeap::print_tracing_info() const {
1182   LogTarget(Info, gc, stats) lt;
1183   if (lt.is_enabled()) {
1184     ResourceMark rm;
1185     LogStream ls(lt);
1186 
1187     phase_timings()->print_global_on(&ls);
1188 
1189     ls.cr();
1190     ls.cr();
1191 
1192     shenandoah_policy()->print_gc_stats(&ls);
1193 
1194     ls.cr();



1195     ls.cr();



















1196   }

1197 }
1198 
1199 void ShenandoahHeap::verify(VerifyOption vo) {
1200   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1201     if (ShenandoahVerify) {
1202       verifier()->verify_generic(vo);
1203     } else {
1204       // TODO: Consider allocating verification bitmaps on demand,
1205       // and turn this on unconditionally.
1206     }
1207   }
1208 }
1209 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1210   return _free_set->capacity();
1211 }
1212 
1213 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1214 private:
1215   MarkBitMap* _bitmap;
1216   ShenandoahScanObjectStack* _oop_stack;

1500       if (start >= max) break;
1501 
1502       for (size_t i = cur; i < end; i++) {
1503         ShenandoahHeapRegion* current = _heap->get_region(i);
1504         _blk->heap_region_do(current);
1505       }
1506     }
1507   }
1508 };
1509 
1510 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1511   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1512   if (num_regions() > ShenandoahParallelRegionStride) {
1513     ShenandoahParallelHeapRegionTask task(blk);
1514     workers()->run_task(&task);
1515   } else {
1516     heap_region_iterate(blk);
1517   }
1518 }
1519 
1520 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1521 private:
1522   ShenandoahMarkingContext* const _ctx;
1523 public:
1524   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1525 
1526   void heap_region_do(ShenandoahHeapRegion* r) {
1527     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1528     if (r->is_active()) {
1529       // Check if region needs updating its TAMS. We have updated it already during concurrent
1530       // reset, so it is very likely we don't need to do another write here.
1531       if (_ctx->top_at_mark_start(r) != r->top()) {
1532         _ctx->capture_top_at_mark_start(r);
1533       }
1534     } else {
1535       assert(_ctx->top_at_mark_start(r) == r->top(),
1536              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1537     }
1538   }
1539 
1540   bool is_thread_safe() { return true; }
1541 };
1542 
1543 class ShenandoahRendezvousClosure : public HandshakeClosure {
1544 public:
1545   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1546   inline void do_thread(Thread* thread) {}
1547 };
1548 
1549 void ShenandoahHeap::rendezvous_threads() {
1550   ShenandoahRendezvousClosure cl;
1551   Handshake::execute(&cl);
1552 }
1553 
1554 void ShenandoahHeap::recycle_trash() {
1555   free_set()->recycle_trash();
1556 }
1557 
1558 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1559 private:
1560   ShenandoahMarkingContext* const _ctx;
1561 public:
1562   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1563 
1564   void heap_region_do(ShenandoahHeapRegion* r) {
1565     if (r->is_active()) {
1566       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1567       // anyway to capture any updates that happened since now.
1568       r->clear_live_data();
1569       _ctx->capture_top_at_mark_start(r);
1570     }
1571   }
1572 
1573   bool is_thread_safe() { return true; }
1574 };
1575 
1576 void ShenandoahHeap::prepare_gc() {
1577   reset_mark_bitmap();
1578 
1579   ShenandoahResetUpdateRegionStateClosure cl;
1580   parallel_heap_region_iterate(&cl);
1581 }
1582 
1583 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1584 private:
1585   ShenandoahMarkingContext* const _ctx;
1586   ShenandoahHeapLock* const _lock;
1587 
1588 public:
1589   ShenandoahFinalMarkUpdateRegionStateClosure() :
1590     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1591 
1592   void heap_region_do(ShenandoahHeapRegion* r) {
1593     if (r->is_active()) {
1594       // All allocations past TAMS are implicitly live, adjust the region data.
1595       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1596       HeapWord *tams = _ctx->top_at_mark_start(r);
1597       HeapWord *top = r->top();
1598       if (top > tams) {
1599         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1600       }
1601 
1602       // We are about to select the collection set, make sure it knows about
1603       // current pinning status. Also, this allows trashing more regions that
1604       // now have their pinning status dropped.
1605       if (r->is_pinned()) {
1606         if (r->pin_count() == 0) {
1607           ShenandoahHeapLocker locker(_lock);
1608           r->make_unpinned();
1609         }
1610       } else {
1611         if (r->pin_count() > 0) {
1612           ShenandoahHeapLocker locker(_lock);
1613           r->make_pinned();
1614         }
1615       }
1616 
1617       // Remember limit for updating refs. It's guaranteed that we get no
1618       // from-space-refs written from here on.
1619       r->set_update_watermark_at_safepoint(r->top());
1620     } else {
1621       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1622       assert(_ctx->top_at_mark_start(r) == r->top(),
1623              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1624     }
1625   }
1626 
1627   bool is_thread_safe() { return true; }
1628 };
1629 
1630 void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
1631   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1632   {
1633     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
1634                                          ShenandoahPhaseTimings::degen_gc_final_update_region_states);
1635     ShenandoahFinalMarkUpdateRegionStateClosure cl;
1636     parallel_heap_region_iterate(&cl);
1637 
1638     assert_pinned_region_status();
1639   }
1640 
1641   {
1642     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
1643                                          ShenandoahPhaseTimings::degen_gc_choose_cset);
1644     ShenandoahHeapLocker locker(lock());
1645     _collection_set->clear();
1646     heuristics()->choose_collection_set(_collection_set);
1647   }
1648 
1649   {
1650     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
1651                                          ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
1652     ShenandoahHeapLocker locker(lock());
1653     _free_set->rebuild();
1654   }
1655 }
1656 
1657 void ShenandoahHeap::do_class_unloading() {
1658   _unloader.unload();



1659 }
1660 
1661 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1662   // Weak refs processing
1663   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1664                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1665   ShenandoahTimingsTracker t(phase);
1666   ShenandoahGCWorkerPhase worker_phase(phase);
1667   ref_processor()->process_references(phase, workers(), false /* concurrent */);
1668 }
1669 
1670 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1671   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1672 
1673   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1674   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1675   // for future GCLABs here.
1676   if (UseTLAB) {
1677     ShenandoahGCPhase phase(concurrent ?
1678                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1679                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1680     gclabs_retire(ResizeTLAB);
1681   }
1682 
1683   _update_refs_iterator.reset();
1684 }
1685 
1686 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1687   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1688   if (_gc_state_changed) {
1689     _gc_state_changed = false;
1690     char state = gc_state();
1691     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1692       ShenandoahThreadLocalData::set_gc_state(t, state);
1693     }
1694   }
1695 }
1696 
1697 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1698   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1699   _gc_state.set_cond(mask, value);
1700   _gc_state_changed = true;
1701 }
1702 
1703 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1704   assert(!has_forwarded_objects(), "Not expected before/after mark phase");
1705   set_gc_state(MARKING, in_progress);
1706   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
















































1707 }
1708 
1709 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1710   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1711   set_gc_state(EVACUATION, in_progress);
1712 }
1713 
1714 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
1715   if (in_progress) {
1716     _concurrent_strong_root_in_progress.set();
1717   } else {
1718     _concurrent_strong_root_in_progress.unset();
1719   }
1720 }
1721 
1722 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
1723   set_gc_state(WEAK_ROOTS, cond);
1724 }
1725 
1726 GCTracer* ShenandoahHeap::tracer() {
1727   return shenandoah_policy()->tracer();
1728 }
1729 
1730 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1731   return _free_set->used();
1732 }
1733 
1734 bool ShenandoahHeap::try_cancel_gc() {
1735   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1736   return prev == CANCELLABLE;
1737 }
1738 








1739 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1740   if (try_cancel_gc()) {
1741     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1742     log_info(gc)("%s", msg.buffer());
1743     Events::log(Thread::current(), "%s", msg.buffer());

1744   }
1745 }
1746 
1747 uint ShenandoahHeap::max_workers() {
1748   return _max_workers;
1749 }
1750 
1751 void ShenandoahHeap::stop() {
1752   // The shutdown sequence should be able to terminate when GC is running.
1753 
1754   // Step 0. Notify policy to disable event recording.
1755   _shenandoah_policy->record_shutdown();
1756 
1757   // Step 1. Notify control thread that we are in shutdown.
1758   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1759   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1760   control_thread()->prepare_for_graceful_shutdown();
1761 
1762   // Step 2. Notify GC workers that we are cancelling GC.
1763   cancel_gc(GCCause::_shenandoah_stop_vm);

1847 }
1848 
1849 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1850   set_gc_state(HAS_FORWARDED, cond);
1851 }
1852 
1853 void ShenandoahHeap::set_unload_classes(bool uc) {
1854   _unload_classes.set_cond(uc);
1855 }
1856 
1857 bool ShenandoahHeap::unload_classes() const {
1858   return _unload_classes.is_set();
1859 }
1860 
1861 address ShenandoahHeap::in_cset_fast_test_addr() {
1862   ShenandoahHeap* heap = ShenandoahHeap::heap();
1863   assert(heap->collection_set() != nullptr, "Sanity");
1864   return (address) heap->collection_set()->biased_map_address();
1865 }
1866 
1867 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
1868   return Atomic::load(&_bytes_allocated_since_gc_start);
1869 }
1870 
1871 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
1872   Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);





1873 }
1874 
1875 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
1876   _degenerated_gc_in_progress.set_cond(in_progress);
1877 }
1878 
1879 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
1880   _full_gc_in_progress.set_cond(in_progress);
1881 }
1882 
1883 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
1884   assert (is_full_gc_in_progress(), "should be");
1885   _full_gc_move_in_progress.set_cond(in_progress);
1886 }
1887 
1888 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
1889   set_gc_state(UPDATEREFS, in_progress);
1890 }
1891 
1892 void ShenandoahHeap::register_nmethod(nmethod* nm) {

1916     if (r->is_active()) {
1917       if (r->is_pinned()) {
1918         if (r->pin_count() == 0) {
1919           r->make_unpinned();
1920         }
1921       } else {
1922         if (r->pin_count() > 0) {
1923           r->make_pinned();
1924         }
1925       }
1926     }
1927   }
1928 
1929   assert_pinned_region_status();
1930 }
1931 
1932 #ifdef ASSERT
1933 void ShenandoahHeap::assert_pinned_region_status() {
1934   for (size_t i = 0; i < num_regions(); i++) {
1935     ShenandoahHeapRegion* r = get_region(i);
1936     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
1937            "Region " SIZE_FORMAT " pinning status is inconsistent", i);


1938   }
1939 }
1940 #endif
1941 
1942 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
1943   return _gc_timer;
1944 }
1945 
1946 void ShenandoahHeap::prepare_concurrent_roots() {
1947   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1948   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1949   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
1950   set_concurrent_weak_root_in_progress(true);
1951   if (unload_classes()) {
1952     _unloader.prepare();
1953   }
1954 }
1955 
1956 void ShenandoahHeap::finish_concurrent_roots() {
1957   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1958   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1959   if (unload_classes()) {
1960     _unloader.finish();
1961   }
1962 }
1963 
1964 #ifdef ASSERT
1965 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
1966   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
1967 
1968   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1969     if (UseDynamicNumberOfGCThreads) {
1970       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
1971     } else {
1972       // Use ParallelGCThreads inside safepoints
1973       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
1974     }
1975   } else {
1976     if (UseDynamicNumberOfGCThreads) {
1977       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
1978     } else {
1979       // Use ConcGCThreads outside safepoints
1980       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
1981     }
1982   }
1983 }
1984 #endif
1985 
1986 ShenandoahVerifier* ShenandoahHeap::verifier() {
1987   guarantee(ShenandoahVerify, "Should be enabled");
1988   assert (_verifier != nullptr, "sanity");
1989   return _verifier;
1990 }
1991 
1992 template<bool CONCURRENT>
1993 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
1994 private:
1995   ShenandoahHeap* _heap;
1996   ShenandoahRegionIterator* _regions;
1997 public:
1998   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
1999     WorkerTask("Shenandoah Update References"),
2000     _heap(ShenandoahHeap::heap()),
2001     _regions(regions) {
2002   }
2003 
2004   void work(uint worker_id) {
2005     if (CONCURRENT) {
2006       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2007       ShenandoahSuspendibleThreadSetJoiner stsj;
2008       do_work<ShenandoahConcUpdateRefsClosure>();
2009     } else {
2010       ShenandoahParallelWorkerSession worker_session(worker_id);
2011       do_work<ShenandoahSTWUpdateRefsClosure>();
2012     }
2013   }
2014 
2015 private:
2016   template<class T>
2017   void do_work() {









2018     T cl;
2019     ShenandoahHeapRegion* r = _regions->next();
2020     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2021     while (r != nullptr) {
2022       HeapWord* update_watermark = r->get_update_watermark();
2023       assert (update_watermark >= r->bottom(), "sanity");
2024       if (r->is_active() && !r->is_cset()) {
2025         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2026       }
2027       if (ShenandoahPacing) {
2028         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2029       }
2030       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2031         return;
2032       }
2033       r = _regions->next();
2034     }
2035   }
2036 };
2037 
2038 void ShenandoahHeap::update_heap_references(bool concurrent) {
2039   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2040 
2041   if (concurrent) {
2042     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2043     workers()->run_task(&task);
2044   } else {
2045     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2046     workers()->run_task(&task);
2047   }
2048 }
2049 
2050 
2051 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2052 private:

2053   ShenandoahHeapLock* const _lock;

2054 
2055 public:
2056   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
























2057 
2058   void heap_region_do(ShenandoahHeapRegion* r) {
2059     // Drop unnecessary "pinned" state from regions that does not have CP marks
2060     // anymore, as this would allow trashing them.
2061 
2062     if (r->is_active()) {
2063       if (r->is_pinned()) {
2064         if (r->pin_count() == 0) {
2065           ShenandoahHeapLocker locker(_lock);
2066           r->make_unpinned();
2067         }
2068       } else {
2069         if (r->pin_count() > 0) {
2070           ShenandoahHeapLocker locker(_lock);
2071           r->make_pinned();
2072         }
2073       }
2074     }
2075   }
2076 
2077   bool is_thread_safe() { return true; }
2078 };
2079 
2080 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2081   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2082   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2083 
2084   {
2085     ShenandoahGCPhase phase(concurrent ?
2086                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2087                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2088     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2089     parallel_heap_region_iterate(&cl);
2090 
2091     assert_pinned_region_status();
2092   }
2093 
2094   {
2095     ShenandoahGCPhase phase(concurrent ?
2096                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2097                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2098     trash_cset_regions();
2099   }
2100 }
2101 
2102 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2103   {
2104     ShenandoahGCPhase phase(concurrent ?
2105                             ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2106                             ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2107     ShenandoahHeapLocker locker(lock());
2108     _free_set->rebuild();




































2109   }
2110 }
2111 
2112 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2113   print_on(st);
2114   st->cr();
2115   print_heap_regions_on(st);
2116 }
2117 
2118 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2119   size_t slice = r->index() / _bitmap_regions_per_slice;
2120 
2121   size_t regions_from = _bitmap_regions_per_slice * slice;
2122   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2123   for (size_t g = regions_from; g < regions_to; g++) {
2124     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2125     if (skip_self && g == r->index()) continue;
2126     if (get_region(g)->is_committed()) {
2127       return true;
2128     }

2176   }
2177 
2178   // Uncommit the bitmap slice:
2179   size_t slice = r->index() / _bitmap_regions_per_slice;
2180   size_t off = _bitmap_bytes_per_slice * slice;
2181   size_t len = _bitmap_bytes_per_slice;
2182   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2183     return false;
2184   }
2185   return true;
2186 }
2187 
2188 void ShenandoahHeap::safepoint_synchronize_begin() {
2189   SuspendibleThreadSet::synchronize();
2190 }
2191 
2192 void ShenandoahHeap::safepoint_synchronize_end() {
2193   SuspendibleThreadSet::desynchronize();
2194 }
2195 
2196 void ShenandoahHeap::entry_uncommit(double shrink_before, size_t shrink_until) {
2197   static const char *msg = "Concurrent uncommit";
2198   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2199   EventMark em("%s", msg);
2200 
2201   op_uncommit(shrink_before, shrink_until);
2202 }
2203 
2204 void ShenandoahHeap::try_inject_alloc_failure() {
2205   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2206     _inject_alloc_failure.set();
2207     os::naked_short_sleep(1);
2208     if (cancelled_gc()) {
2209       log_info(gc)("Allocation failure was successfully injected");
2210     }
2211   }
2212 }
2213 
2214 bool ShenandoahHeap::should_inject_alloc_failure() {
2215   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2216 }
2217 
2218 void ShenandoahHeap::initialize_serviceability() {
2219   _memory_pool = new ShenandoahMemoryPool(this);
2220   _cycle_memory_manager.add_pool(_memory_pool);
2221   _stw_memory_manager.add_pool(_memory_pool);
2222 }
2223 
2224 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2225   GrowableArray<GCMemoryManager*> memory_managers(2);
2226   memory_managers.append(&_cycle_memory_manager);
2227   memory_managers.append(&_stw_memory_manager);
2228   return memory_managers;
2229 }
2230 
2231 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2232   GrowableArray<MemoryPool*> memory_pools(1);
2233   memory_pools.append(_memory_pool);
2234   return memory_pools;
2235 }
2236 
2237 MemoryUsage ShenandoahHeap::memory_usage() {
2238   return _memory_pool->get_memory_usage();
2239 }
2240 
2241 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2242   _heap(ShenandoahHeap::heap()),
2243   _index(0) {}
2244 
2245 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2246   _heap(heap),
2247   _index(0) {}
2248 
2249 void ShenandoahRegionIterator::reset() {
2250   _index = 0;
2251 }
2252 
2253 bool ShenandoahRegionIterator::has_next() const {
2254   return _index < _heap->num_regions();
2255 }
2256 
2257 char ShenandoahHeap::gc_state() const {
2258   return _gc_state.raw_value();

2283   }
2284 }
2285 
2286 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2287   if (is_idle()) return false;
2288 
2289   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2290   // marking phase.
2291   if (is_concurrent_mark_in_progress() &&
2292      !marking_context()->allocated_after_mark_start(obj)) {
2293     return true;
2294   }
2295 
2296   // Can not guarantee obj is deeply good.
2297   if (has_forwarded_objects()) {
2298     return true;
2299   }
2300 
2301   return false;
2302 }



































   1 /*
   2  * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
   4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "memory/allocation.hpp"
  29 #include "memory/universe.hpp"
  30 
  31 #include "gc/shared/classUnloadingContext.hpp"
  32 #include "gc/shared/gcArguments.hpp"
  33 #include "gc/shared/gcTimer.hpp"
  34 #include "gc/shared/gcTraceTime.inline.hpp"
  35 #include "gc/shared/locationPrinter.inline.hpp"
  36 #include "gc/shared/memAllocator.hpp"
  37 #include "gc/shared/plab.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 
  40 #include "gc/shenandoah/shenandoahAgeCensus.hpp"
  41 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
  42 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
  43 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  44 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  45 #include "gc/shenandoah/shenandoahCardTable.hpp"
  46 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  47 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  48 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  49 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  50 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  51 #include "gc/shenandoah/shenandoahControlThread.hpp"
  52 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  53 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
  54 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  55 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
  56 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  57 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  58 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  59 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  60 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  61 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  62 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  63 #include "gc/shenandoah/shenandoahMetrics.hpp"
  64 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  65 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  66 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  67 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  68 #include "gc/shenandoah/shenandoahPadding.hpp"
  69 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  70 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  71 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  72 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  73 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  74 #include "gc/shenandoah/shenandoahUtils.hpp"
  75 #include "gc/shenandoah/shenandoahVerifier.hpp"
  76 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  77 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  78 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  79 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  80 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  81 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
  82 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  83 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  84 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  85 #include "utilities/globalDefinitions.hpp"
  86 
  87 #if INCLUDE_JFR
  88 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  89 #endif
  90 
  91 #include "classfile/systemDictionary.hpp"
  92 #include "code/codeCache.hpp"
  93 #include "memory/classLoaderMetaspace.hpp"
  94 #include "memory/metaspaceUtils.hpp"
  95 #include "oops/compressedOops.inline.hpp"
  96 #include "prims/jvmtiTagMap.hpp"
  97 #include "runtime/atomic.hpp"
  98 #include "runtime/globals.hpp"
  99 #include "runtime/interfaceSupport.inline.hpp"
 100 #include "runtime/java.hpp"
 101 #include "runtime/orderAccess.hpp"
 102 #include "runtime/safepointMechanism.hpp"
 103 #include "runtime/vmThread.hpp"
 104 #include "services/mallocTracker.hpp"
 105 #include "services/memTracker.hpp"
 106 #include "utilities/events.hpp"

 158 jint ShenandoahHeap::initialize() {
 159   //
 160   // Figure out heap sizing
 161   //
 162 
 163   size_t init_byte_size = InitialHeapSize;
 164   size_t min_byte_size  = MinHeapSize;
 165   size_t max_byte_size  = MaxHeapSize;
 166   size_t heap_alignment = HeapAlignment;
 167 
 168   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 169 
 170   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 171   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 172 
 173   _num_regions = ShenandoahHeapRegion::region_count();
 174   assert(_num_regions == (max_byte_size / reg_size_bytes),
 175          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 176          _num_regions, max_byte_size, reg_size_bytes);
 177 



 178   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 179   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 180   assert(num_committed_regions <= _num_regions, "sanity");
 181   _initial_size = num_committed_regions * reg_size_bytes;
 182 
 183   size_t num_min_regions = min_byte_size / reg_size_bytes;
 184   num_min_regions = MIN2(num_min_regions, _num_regions);
 185   assert(num_min_regions <= _num_regions, "sanity");
 186   _minimum_size = num_min_regions * reg_size_bytes;
 187 
 188   // Default to max heap size.
 189   _soft_max_size = _num_regions * reg_size_bytes;
 190 
 191   _committed = _initial_size;
 192 
 193   // Now we know the number of regions and heap sizes, initialize the heuristics.
 194   initialize_heuristics_generations();
 195 
 196   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 197   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 198   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 199 
 200   //
 201   // Reserve and commit memory for heap
 202   //
 203 
 204   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 205   initialize_reserved_region(heap_rs);
 206   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 207   _heap_region_special = heap_rs.special();
 208 
 209   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 210          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 211   os::trace_page_sizes_for_requested_size("Heap",
 212                                           max_byte_size, heap_rs.page_size(), heap_alignment,
 213                                           heap_rs.base(), heap_rs.size());
 214 
 215 #if SHENANDOAH_OPTIMIZED_MARKTASK
 216   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 217   // Fail if we ever attempt to address more than we can.
 218   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 219     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 220                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 221                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 222                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 223     vm_exit_during_initialization("Fatal Error", buf);
 224   }
 225 #endif
 226 
 227   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 228   if (!_heap_region_special) {
 229     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 230                               "Cannot commit heap memory");
 231   }
 232 
 233   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
 234 
 235   //
 236   // After reserving the Java heap, create the card table, barriers, and workers, in dependency order
 237   //
 238   if (mode()->is_generational()) {
 239     ShenandoahDirectCardMarkRememberedSet *rs;
 240     ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table();
 241     size_t card_count = card_table->cards_required(heap_rs.size() / HeapWordSize);
 242     rs = new ShenandoahDirectCardMarkRememberedSet(ShenandoahBarrierSet::barrier_set()->card_table(), card_count);
 243     _card_scan = new ShenandoahScanRemembered<ShenandoahDirectCardMarkRememberedSet>(rs);
 244 
 245     // Age census structure
 246     _age_census = new ShenandoahAgeCensus();
 247   }
 248 
 249   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 250   if (_workers == nullptr) {
 251     vm_exit_during_initialization("Failed necessary allocation.");
 252   } else {
 253     _workers->initialize_workers();
 254   }
 255 
 256   if (ParallelGCThreads > 1) {
 257     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread", ParallelGCThreads);
 258     _safepoint_workers->initialize_workers();
 259   }
 260 
 261   //
 262   // Reserve and commit memory for bitmap(s)
 263   //
 264 
 265   size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 266   _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
 267 
 268   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 269 
 270   guarantee(bitmap_bytes_per_region != 0,
 271             "Bitmap bytes per region should not be zero");
 272   guarantee(is_power_of_2(bitmap_bytes_per_region),
 273             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 274 
 275   if (bitmap_page_size > bitmap_bytes_per_region) {
 276     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 277     _bitmap_bytes_per_slice = bitmap_page_size;
 278   } else {
 279     _bitmap_regions_per_slice = 1;
 280     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 281   }
 282 
 283   guarantee(_bitmap_regions_per_slice >= 1,
 284             "Should have at least one region per slice: " SIZE_FORMAT,
 285             _bitmap_regions_per_slice);
 286 
 287   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 288             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 289             _bitmap_bytes_per_slice, bitmap_page_size);
 290 
 291   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 292   os::trace_page_sizes_for_requested_size("Mark Bitmap",
 293                                           bitmap_size_orig, bitmap.page_size(), bitmap_page_size,
 294                                           bitmap.base(),
 295                                           bitmap.size());
 296   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 297   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 298   _bitmap_region_special = bitmap.special();
 299 
 300   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 301     align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 302   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 303   if (!_bitmap_region_special) {
 304     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 305                               "Cannot commit bitmap memory");
 306   }
 307 
 308   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 309 
 310   if (ShenandoahVerify) {
 311     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 312     os::trace_page_sizes_for_requested_size("Verify Bitmap",
 313                                             bitmap_size_orig, verify_bitmap.page_size(), bitmap_page_size,
 314                                             verify_bitmap.base(),
 315                                             verify_bitmap.size());
 316     if (!verify_bitmap.special()) {
 317       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 318                                 "Cannot commit verification bitmap memory");
 319     }
 320     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 321     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 322     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 323     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 324   }
 325 
 326   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 327   size_t aux_bitmap_page_size = bitmap_page_size;
 328 #ifdef LINUX
 329   // In THP "advise" mode, we refrain from advising the system to use large pages
 330   // since we know these commits will be short lived, and there is no reason to trash
 331   // the THP area with this bitmap.
 332   if (UseTransparentHugePages) {
 333     aux_bitmap_page_size = os::vm_page_size();
 334   }
 335 #endif
 336   ReservedSpace aux_bitmap(_bitmap_size, aux_bitmap_page_size);
 337   os::trace_page_sizes_for_requested_size("Aux Bitmap",
 338                                           bitmap_size_orig, aux_bitmap.page_size(), aux_bitmap_page_size,
 339                                           aux_bitmap.base(), aux_bitmap.size());
 340   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 341   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 342   _aux_bitmap_region_special = aux_bitmap.special();
 343   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 344 
 345   //
 346   // Create regions and region sets
 347   //
 348   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 349   size_t region_storage_size_orig = region_align * _num_regions;
 350   size_t region_storage_size = align_up(region_storage_size_orig,
 351                                         MAX2(region_page_size, os::vm_allocation_granularity()));
 352 
 353   ReservedSpace region_storage(region_storage_size, region_page_size);
 354   os::trace_page_sizes_for_requested_size("Region Storage",
 355                                           region_storage_size_orig, region_storage.page_size(), region_page_size,
 356                                           region_storage.base(), region_storage.size());
 357   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 358   if (!region_storage.special()) {
 359     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 360                               "Cannot commit region memory");
 361   }
 362 
 363   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 364   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 365   // If not successful, bite a bullet and allocate at whatever address.
 366   {
 367     const size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 368     const size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 369     const size_t cset_page_size = os::vm_page_size();
 370 
 371     uintptr_t min = round_up_power_of_2(cset_align);
 372     uintptr_t max = (1u << 30u);
 373     ReservedSpace cset_rs;
 374 
 375     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 376       char* req_addr = (char*)addr;
 377       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 378       cset_rs = ReservedSpace(cset_size, cset_align, cset_page_size, req_addr);
 379       if (cset_rs.is_reserved()) {
 380         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 381         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 382         break;
 383       }
 384     }
 385 
 386     if (_collection_set == nullptr) {
 387       cset_rs = ReservedSpace(cset_size, cset_align, os::vm_page_size());
 388       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 389     }
 390     os::trace_page_sizes_for_requested_size("Collection Set",
 391                                             cset_size, cset_rs.page_size(), cset_page_size,
 392                                             cset_rs.base(),
 393                                             cset_rs.size());
 394   }
 395 
 396   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 397   _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC);
 398   _free_set = new ShenandoahFreeSet(this, _num_regions);
 399 
 400   {
 401     ShenandoahHeapLocker locker(lock());
 402 
 403 
 404     for (size_t i = 0; i < _num_regions; i++) {
 405       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 406       bool is_committed = i < num_committed_regions;
 407       void* loc = region_storage.base() + i * region_align;
 408 
 409       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 410       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 411 
 412       _marking_context->initialize_top_at_mark_start(r);
 413       _regions[i] = r;
 414       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 415 
 416       _affiliations[i] = ShenandoahAffiliation::FREE;
 417     }
 418 
 419     // Initialize to complete
 420     _marking_context->mark_complete();
 421     size_t young_cset_regions, old_cset_regions;
 422 
 423     // We are initializing free set.  We ignore cset region tallies.
 424     size_t first_old, last_old, num_old;
 425     _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
 426     _free_set->rebuild(young_cset_regions, old_cset_regions);
 427   }
 428 
 429   if (AlwaysPreTouch) {
 430     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 431     // before initialize() below zeroes it with initializing thread. For any given region,
 432     // we touch the region and the corresponding bitmaps from the same thread.
 433     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 434 
 435     _pretouch_heap_page_size = heap_page_size;
 436     _pretouch_bitmap_page_size = bitmap_page_size;
 437 
 438 #ifdef LINUX
 439     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 440     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 441     // them into huge one. Therefore, we need to pretouch with smaller pages.
 442     if (UseTransparentHugePages) {
 443       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 444       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 445     }
 446 #endif

 463   for (uint worker = 0; worker < _max_workers; worker++) {
 464     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 465     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 466   }
 467 
 468   // There should probably be Shenandoah-specific options for these,
 469   // just as there are G1-specific options.
 470   {
 471     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 472     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 473     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 474   }
 475 
 476   _monitoring_support = new ShenandoahMonitoringSupport(this);
 477   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 478   ShenandoahCodeRoots::initialize();
 479 
 480   if (ShenandoahPacing) {
 481     _pacer = new ShenandoahPacer(this);
 482     _pacer->setup_for_idle();


 483   }
 484 
 485   initialize_controller();
 486 
 487   print_init_logger();
 488 
 489   return JNI_OK;
 490 }
 491 
 492 void ShenandoahHeap::initialize_controller() {
 493   _control_thread = new ShenandoahControlThread();
 494 }
 495 
 496 void ShenandoahHeap::print_init_logger() const {
 497   ShenandoahInitLogger::print();
 498 }
 499 
 500 void ShenandoahHeap::initialize_heuristics_generations() {
 501   if (ShenandoahGCMode != nullptr) {
 502     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 503       _gc_mode = new ShenandoahSATBMode();
 504     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 505       _gc_mode = new ShenandoahIUMode();
 506     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 507       _gc_mode = new ShenandoahPassiveMode();
 508     } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
 509       _gc_mode = new ShenandoahGenerationalMode();
 510     } else {
 511       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 512     }
 513   } else {
 514     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 515   }
 516   _gc_mode->initialize_flags();
 517   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 518     vm_exit_during_initialization(
 519             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 520                     _gc_mode->name()));
 521   }
 522   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 523     vm_exit_during_initialization(
 524             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 525                     _gc_mode->name()));
 526   }

 527 
 528   // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity
 529   // for old would be total heap - minimum capacity of young. This means the sum of the maximum
 530   // allowed for old and young could exceed the total heap size. It remains the case that the
 531   // _actual_ capacity of young + old = total.
 532   _generation_sizer.heap_size_changed(max_capacity());
 533   size_t initial_capacity_young = _generation_sizer.max_young_size();
 534   size_t max_capacity_young = _generation_sizer.max_young_size();
 535   size_t initial_capacity_old = max_capacity() - max_capacity_young;
 536   size_t max_capacity_old = max_capacity() - initial_capacity_young;
 537 
 538   _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_young, initial_capacity_young);
 539   _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, initial_capacity_old);
 540   _global_generation = new ShenandoahGlobalGeneration(_gc_mode->is_generational(), _max_workers, max_capacity(), max_capacity());
 541   _global_generation->initialize_heuristics(_gc_mode);
 542   if (mode()->is_generational()) {
 543     _young_generation->initialize_heuristics(_gc_mode);
 544     _old_generation->initialize_heuristics(_gc_mode);


 545   }
 546   _evac_tracker = new ShenandoahEvacuationTracker(mode()->is_generational());
 547 }
 548 
 549 #ifdef _MSC_VER
 550 #pragma warning( push )
 551 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 552 #endif
 553 
 554 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 555   CollectedHeap(),
 556   _gc_generation(nullptr),
 557   _initial_size(0),

 558   _committed(0),
 559   _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),

 560   _workers(nullptr),
 561   _safepoint_workers(nullptr),
 562   _heap_region_special(false),
 563   _num_regions(0),
 564   _regions(nullptr),
 565   _affiliations(nullptr),
 566   _gc_state_changed(false),
 567   _gc_no_progress_count(0),
 568   _age_census(nullptr),
 569   _cancel_requested_time(0),
 570   _update_refs_iterator(this),
 571   _young_generation(nullptr),
 572   _global_generation(nullptr),
 573   _old_generation(nullptr),
 574   _control_thread(nullptr),
 575   _shenandoah_policy(policy),


 576   _free_set(nullptr),
 577   _pacer(nullptr),
 578   _verifier(nullptr),
 579   _phase_timings(nullptr),
 580   _evac_tracker(nullptr),
 581   _mmu_tracker(),
 582   _generation_sizer(),
 583   _monitoring_support(nullptr),
 584   _memory_pool(nullptr),
 585   _stw_memory_manager("Shenandoah Pauses"),
 586   _cycle_memory_manager("Shenandoah Cycles"),
 587   _gc_timer(new ConcurrentGCTimer()),
 588   _soft_ref_policy(),
 589   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),

 590   _marking_context(nullptr),
 591   _bitmap_size(0),
 592   _bitmap_regions_per_slice(0),
 593   _bitmap_bytes_per_slice(0),
 594   _bitmap_region_special(false),
 595   _aux_bitmap_region_special(false),
 596   _liveness_cache(nullptr),
 597   _collection_set(nullptr),
 598   _card_scan(nullptr)
 599 {

















 600 }
 601 
 602 #ifdef _MSC_VER
 603 #pragma warning( pop )
 604 #endif
 605 





























 606 void ShenandoahHeap::print_on(outputStream* st) const {
 607   st->print_cr("Shenandoah Heap");
 608   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 609                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 610                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 611                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 612                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 613   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 614                num_regions(),
 615                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 616                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 617 
 618   st->print("Status: ");
 619   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 620   if (is_concurrent_old_mark_in_progress())    st->print("old marking, ");
 621   if (is_concurrent_young_mark_in_progress())  st->print("young marking, ");
 622   if (is_evacuation_in_progress())             st->print("evacuating, ");
 623   if (is_update_refs_in_progress())            st->print("updating refs, ");
 624   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 625   if (is_full_gc_in_progress())                st->print("full gc, ");
 626   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 627   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 628   if (is_concurrent_strong_root_in_progress() &&
 629       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 630 
 631   if (cancelled_gc()) {
 632     st->print("cancelled");
 633   } else {
 634     st->print("not cancelled");
 635   }
 636   st->cr();
 637 
 638   st->print_cr("Reserved region:");
 639   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 640                p2i(reserved_region().start()),
 641                p2i(reserved_region().end()));

 652   st->cr();
 653   MetaspaceUtils::print_on(st);
 654 
 655   if (Verbose) {
 656     st->cr();
 657     print_heap_regions_on(st);
 658   }
 659 }
 660 
 661 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 662 public:
 663   void do_thread(Thread* thread) {
 664     assert(thread != nullptr, "Sanity");
 665     assert(thread->is_Worker_thread(), "Only worker thread expected");
 666     ShenandoahThreadLocalData::initialize_gclab(thread);
 667   }
 668 };
 669 
 670 void ShenandoahHeap::post_initialize() {
 671   CollectedHeap::post_initialize();
 672   _mmu_tracker.initialize();
 673 
 674   MutexLocker ml(Threads_lock);
 675 
 676   ShenandoahInitWorkerGCLABClosure init_gclabs;
 677   _workers->threads_do(&init_gclabs);
 678 
 679   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 680   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 681   _workers->set_initialize_gclab();
 682   if (_safepoint_workers != nullptr) {
 683     _safepoint_workers->threads_do(&init_gclabs);
 684     _safepoint_workers->set_initialize_gclab();
 685   }
 686 


 687   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 688 }
 689 
 690 ShenandoahHeuristics* ShenandoahHeap::heuristics() {
 691   return _global_generation->heuristics();
 692 }
 693 
 694 size_t ShenandoahHeap::used() const {
 695   return global_generation()->used();
 696 }
 697 
 698 size_t ShenandoahHeap::committed() const {
 699   return Atomic::load(&_committed);
 700 }
 701 
 702 void ShenandoahHeap::increase_committed(size_t bytes) {
 703   shenandoah_assert_heaplocked_or_safepoint();
 704   _committed += bytes;
 705 }
 706 
 707 void ShenandoahHeap::decrease_committed(size_t bytes) {
 708   shenandoah_assert_heaplocked_or_safepoint();
 709   _committed -= bytes;
 710 }
 711 
 712 // For tracking usage based on allocations, it should be the case that:
 713 // * The sum of regions::used == heap::used
 714 // * The sum of a generation's regions::used == generation::used
 715 // * The sum of a generation's humongous regions::free == generation::humongous_waste
 716 // These invariants are checked by the verifier on GC safepoints.
 717 //
 718 // Additional notes:
 719 // * When a mutator's allocation request causes a region to be retired, the
 720 //   free memory left in that region is considered waste. It does not contribute
 721 //   to the usage, but it _does_ contribute to allocation rate.
 722 // * The bottom of a PLAB must be aligned on card size. In some cases this will
 723 //   require padding in front of the PLAB (a filler object). Because this padding
 724 //   is included in the region's used memory we include the padding in the usage
 725 //   accounting as waste.
 726 // * Mutator allocations are used to compute an allocation rate. They are also
 727 //   sent to the Pacer for those purposes.
 728 // * There are three sources of waste:
 729 //  1. The padding used to align a PLAB on card size
 730 //  2. Region's free is less than minimum TLAB size and is retired
 731 //  3. The unused portion of memory in the last region of a humongous object
 732 void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) {
 733   size_t actual_bytes = req.actual_size() * HeapWordSize;
 734   size_t wasted_bytes = req.waste() * HeapWordSize;
 735   ShenandoahGeneration* generation = generation_for(req.affiliation());
 736 
 737   if (req.is_gc_alloc()) {
 738     assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste");
 739     increase_used(generation, actual_bytes + wasted_bytes);
 740   } else {
 741     assert(req.is_mutator_alloc(), "Expected mutator alloc here");
 742     // padding and actual size both count towards allocation counter
 743     generation->increase_allocated(actual_bytes + wasted_bytes);
 744 
 745     // only actual size counts toward usage for mutator allocations
 746     increase_used(generation, actual_bytes);
 747 
 748     // notify pacer of both actual size and waste
 749     notify_mutator_alloc_words(req.actual_size(), req.waste());
 750 
 751     if (wasted_bytes > 0 && req.actual_size() > ShenandoahHeapRegion::humongous_threshold_words()) {
 752       increase_humongous_waste(generation,wasted_bytes);
 753     }
 754   }
 755 }
 756 
 757 void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 758   generation->increase_humongous_waste(bytes);
 759   if (!generation->is_global()) {
 760     global_generation()->increase_humongous_waste(bytes);
 761   }
 762 }
 763 
 764 void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 765   generation->decrease_humongous_waste(bytes);
 766   if (!generation->is_global()) {
 767     global_generation()->decrease_humongous_waste(bytes);
 768   }
 769 }
 770 
 771 void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) {
 772   generation->increase_used(bytes);
 773   if (!generation->is_global()) {
 774     global_generation()->increase_used(bytes);
 775   }
 776 }
 777 
 778 void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) {
 779   generation->decrease_used(bytes);
 780   if (!generation->is_global()) {
 781     global_generation()->decrease_used(bytes);
 782   }
 783 }
 784 
 785 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) {
 786   if (ShenandoahPacing) {
 787     control_thread()->pacing_notify_alloc(words);
 788     if (waste > 0) {
 789       pacer()->claim_for_alloc(waste, true);
 790     }
 791   }
 792 }
 793 
 794 size_t ShenandoahHeap::capacity() const {
 795   return committed();
 796 }
 797 
 798 size_t ShenandoahHeap::max_capacity() const {
 799   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 800 }
 801 
 802 size_t ShenandoahHeap::soft_max_capacity() const {
 803   size_t v = Atomic::load(&_soft_max_size);
 804   assert(min_capacity() <= v && v <= max_capacity(),
 805          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 806          min_capacity(), v, max_capacity());
 807   return v;
 808 }
 809 
 810 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 811   assert(min_capacity() <= v && v <= max_capacity(),
 812          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 813          min_capacity(), v, max_capacity());
 814   Atomic::store(&_soft_max_size, v);
 815 }
 816 
 817 size_t ShenandoahHeap::min_capacity() const {
 818   return _minimum_size;
 819 }
 820 
 821 size_t ShenandoahHeap::initial_capacity() const {
 822   return _initial_size;
 823 }
 824 
 825 void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
 826   assert (ShenandoahUncommit, "should be enabled");
 827 
 828   // Determine if there is work to do. This avoids taking heap lock if there is
 829   // no work available, avoids spamming logs with superfluous logging messages,
 830   // and minimises the amount of work while locks are taken.
 831 
 832   if (committed() <= shrink_until) return;
 833 
 834   bool has_work = false;
 835   for (size_t i = 0; i < num_regions(); i++) {
 836     ShenandoahHeapRegion* r = get_region(i);
 837     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 838       has_work = true;
 839       break;
 840     }
 841   }
 842 
 843   if (has_work) {
 844     static const char* msg = "Concurrent uncommit";
 845     ShenandoahConcurrentPhase gcPhase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
 846     EventMark em("%s", msg);
 847 
 848     op_uncommit(shrink_before, shrink_until);
 849   }
 850 }
 851 
 852 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
 853   assert (ShenandoahUncommit, "should be enabled");
 854 
 855   // Application allocates from the beginning of the heap, and GC allocates at
 856   // the end of it. It is more efficient to uncommit from the end, so that applications
 857   // could enjoy the near committed regions. GC allocations are much less frequent,
 858   // and therefore can accept the committing costs.
 859 
 860   size_t count = 0;
 861   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 862     ShenandoahHeapRegion* r = get_region(i - 1);
 863     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 864       ShenandoahHeapLocker locker(lock());
 865       if (r->is_empty_committed()) {
 866         if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
 867           break;
 868         }
 869 
 870         r->make_uncommitted();
 871         count++;
 872       }
 873     }
 874     SpinPause(); // allow allocators to take the lock
 875   }
 876 
 877   if (count > 0) {
 878     notify_heap_changed();
 879   }
 880 }
 881 
 882 bool ShenandoahHeap::check_soft_max_changed() {
 883   size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
 884   size_t old_soft_max = soft_max_capacity();
 885   if (new_soft_max != old_soft_max) {
 886     new_soft_max = MAX2(min_capacity(), new_soft_max);
 887     new_soft_max = MIN2(max_capacity(), new_soft_max);
 888     if (new_soft_max != old_soft_max) {
 889       log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
 890                    byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
 891                    byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
 892       );
 893       set_soft_max_capacity(new_soft_max);
 894       return true;
 895     }
 896   }
 897   return false;
 898 }
 899 
 900 void ShenandoahHeap::notify_heap_changed() {
 901   // Update monitoring counters when we took a new region. This amortizes the
 902   // update costs on slow path.
 903   monitoring_support()->notify_heap_changed();
 904   _heap_changed.set();
 905 }
 906 
 907 void ShenandoahHeap::set_forced_counters_update(bool value) {
 908   monitoring_support()->set_forced_counters_update(value);
 909 }
 910 
 911 void ShenandoahHeap::handle_force_counters_update() {
 912   monitoring_support()->handle_force_counters_update();
 913 }
 914 
 915 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 916   // New object should fit the GCLAB size
 917   size_t min_size = MAX2(size, PLAB::min_size());
 918 
 919   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 920   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 921 
 922   new_size = MIN2(new_size, PLAB::max_size());
 923   new_size = MAX2(new_size, PLAB::min_size());
 924 
 925   // Record new heuristic value even if we take any shortcut. This captures
 926   // the case when moderately-sized objects always take a shortcut. At some point,
 927   // heuristics should catch up with them.
 928   log_debug(gc, free)("Set new GCLAB size: " SIZE_FORMAT, new_size);
 929   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 930 
 931   if (new_size < size) {
 932     // New size still does not fit the object. Fall back to shared allocation.
 933     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 934     log_debug(gc, free)("New gclab size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, new_size, size);
 935     return nullptr;
 936   }
 937 
 938   // Retire current GCLAB, and allocate a new one.
 939   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 940   gclab->retire();
 941 
 942   size_t actual_size = 0;
 943   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 944   if (gclab_buf == nullptr) {
 945     return nullptr;
 946   }
 947 
 948   assert (size <= actual_size, "allocation should fit");
 949 
 950   if (ZeroTLAB) {
 951     // ..and clear it.
 952     Copy::zero_to_words(gclab_buf, actual_size);
 953   } else {
 954     // ...and zap just allocated object.
 955 #ifdef ASSERT
 956     // Skip mangling the space corresponding to the object header to
 957     // ensure that the returned space is not considered parsable by
 958     // any concurrent GC thread.
 959     size_t hdr_size = oopDesc::header_size();
 960     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 961 #endif // ASSERT
 962   }
 963   gclab->set_buf(gclab_buf, actual_size);
 964   return gclab->allocate(size);
 965 }
 966 
 967 // Called from stubs in JIT code or interpreter
 968 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 969                                             size_t requested_size,
 970                                             size_t* actual_size) {
 971   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 972   HeapWord* res = allocate_memory(req);
 973   if (res != nullptr) {
 974     *actual_size = req.actual_size();
 975   } else {
 976     *actual_size = 0;
 977   }
 978   return res;
 979 }
 980 
 981 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 982                                              size_t word_size,
 983                                              size_t* actual_size) {
 984   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 985   HeapWord* res = allocate_memory(req);
 986   if (res != nullptr) {
 987     *actual_size = req.actual_size();

 989     *actual_size = 0;
 990   }
 991   return res;
 992 }
 993 
 994 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 995   intptr_t pacer_epoch = 0;
 996   bool in_new_region = false;
 997   HeapWord* result = nullptr;
 998 
 999   if (req.is_mutator_alloc()) {
1000     if (ShenandoahPacing) {
1001       pacer()->pace_for_alloc(req.size());
1002       pacer_epoch = pacer()->epoch();
1003     }
1004 
1005     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
1006       result = allocate_memory_under_lock(req, in_new_region);
1007     }
1008 
1009     // Check that gc overhead is not exceeded.
1010     //
1011     // Shenandoah will grind along for quite a while allocating one
1012     // object at a time using shared (non-tlab) allocations. This check
1013     // is testing that the GC overhead limit has not been exceeded.
1014     // This will notify the collector to start a cycle, but will raise
1015     // an OOME to the mutator if the last Full GCs have not made progress.
1016     if (result == nullptr && !req.is_lab_alloc() && get_gc_no_progress_count() > ShenandoahNoProgressThreshold) {
1017       control_thread()->handle_alloc_failure(req, false);
1018       return nullptr;
1019     }
1020 
1021     // Block until control thread reacted, then retry allocation.
1022     //
1023     // It might happen that one of the threads requesting allocation would unblock
1024     // way later after GC happened, only to fail the second allocation, because
1025     // other threads have already depleted the free storage. In this case, a better
1026     // strategy is to try again, as long as GC makes progress (or until at least
1027     // one full GC has completed).
1028     size_t original_count = shenandoah_policy()->full_gc_count();
1029     while (result == nullptr
1030         && (get_gc_no_progress_count() == 0 || original_count == shenandoah_policy()->full_gc_count())) {
1031       control_thread()->handle_alloc_failure(req, true);
1032       result = allocate_memory_under_lock(req, in_new_region);
1033     }
1034 
1035     if (log_is_enabled(Debug, gc, alloc)) {
1036       ResourceMark rm;
1037       log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT ", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
1038                            Thread::current()->name(), p2i(result), req.type_string(), req.size(), original_count, get_gc_no_progress_count());
1039     }
1040   } else {
1041     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
1042     result = allocate_memory_under_lock(req, in_new_region);
1043     // Do not call handle_alloc_failure() here, because we cannot block.
1044     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
1045   }
1046 
1047   if (in_new_region) {
1048     notify_heap_changed();
1049   }
1050 
1051   if (result == nullptr) {
1052     req.set_actual_size(0);
1053   }
1054 
1055   // This is called regardless of the outcome of the allocation to account
1056   // for any waste created by retiring regions with this request.
1057   increase_used(req);
1058 
1059   if (result != nullptr) {
1060     size_t requested = req.size();
1061     size_t actual = req.actual_size();
1062 
1063     assert (req.is_lab_alloc() || (requested == actual),
1064             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
1065             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
1066 
1067     if (req.is_mutator_alloc()) {


1068       // If we requested more than we were granted, give the rest back to pacer.
1069       // This only matters if we are in the same pacing epoch: do not try to unpace
1070       // over the budget for the other phase.
1071       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
1072         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
1073       }


1074     }
1075   }
1076 
1077   return result;
1078 }
1079 
1080 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1081   // If we are dealing with mutator allocation, then we may need to block for safepoint.
1082   // We cannot block for safepoint for GC allocations, because there is a high chance
1083   // we are already running at safepoint or from stack watermark machinery, and we cannot
1084   // block again.
1085   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1086 
1087   // Make sure the old generation has room for either evacuations or promotions before trying to allocate.
1088   if (req.is_old() && !old_generation()->can_allocate(req)) {
1089     return nullptr;
1090   }
1091 
1092   // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
1093   // memory.
1094   HeapWord* result = _free_set->allocate(req, in_new_region);
1095 
1096   // Record the plab configuration for this result and register the object.
1097   if (result != nullptr && req.is_old()) {
1098     old_generation()->configure_plab_for_current_thread(req);
1099     if (req.type() == ShenandoahAllocRequest::_alloc_shared_gc) {
1100       // Register the newly allocated object while we're holding the global lock since there's no synchronization
1101       // built in to the implementation of register_object().  There are potential races when multiple independent
1102       // threads are allocating objects, some of which might span the same card region.  For example, consider
1103       // a card table's memory region within which three objects are being allocated by three different threads:
1104       //
1105       // objects being "concurrently" allocated:
1106       //    [-----a------][-----b-----][--------------c------------------]
1107       //            [---- card table memory range --------------]
1108       //
1109       // Before any objects are allocated, this card's memory range holds no objects.  Note that allocation of object a
1110       // wants to set the starts-object, first-start, and last-start attributes of the preceding card region.
1111       // Allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region.
1112       // Allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this
1113       // card region.
1114       //
1115       // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as
1116       // last-start representing object b while first-start represents object c.  This is why we need to require all
1117       // register_object() invocations to be "mutually exclusive" with respect to each card's memory range.
1118       card_scan()->register_object(result);
1119     }
1120   }
1121 
1122   return result;
1123 }
1124 
1125 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1126                                         bool*  gc_overhead_limit_was_exceeded) {
1127   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1128   return allocate_memory(req);
1129 }
1130 
1131 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1132                                                              size_t size,
1133                                                              Metaspace::MetadataType mdtype) {
1134   MetaWord* result;
1135 
1136   // Inform metaspace OOM to GC heuristics if class unloading is possible.
1137   ShenandoahHeuristics* h = global_generation()->heuristics();
1138   if (h->can_unload_classes()) {
1139     h->record_metaspace_oom();
1140   }
1141 
1142   // Expand and retry allocation
1143   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1144   if (result != nullptr) {
1145     return result;
1146   }
1147 
1148   // Start full GC
1149   collect(GCCause::_metadata_GC_clear_soft_refs);
1150 
1151   // Retry allocation
1152   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1153   if (result != nullptr) {
1154     return result;
1155   }
1156 
1157   // Expand and retry allocation
1158   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

1211 private:
1212   void do_work() {
1213     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1214     ShenandoahHeapRegion* r;
1215     while ((r =_cs->claim_next()) != nullptr) {
1216       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
1217       _sh->marked_object_iterate(r, &cl);
1218 
1219       if (ShenandoahPacing) {
1220         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1221       }
1222 
1223       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1224         break;
1225       }
1226     }
1227   }
1228 };
1229 
1230 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1231   if (mode()->is_generational()) {
1232     ShenandoahRegionIterator regions;
1233     ShenandoahGenerationalEvacuationTask task(ShenandoahGenerationalHeap::heap(), &regions, concurrent);
1234     workers()->run_task(&task);
1235   } else {
1236     ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1237     workers()->run_task(&task);
1238   }
1239 }
1240 
1241 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1242   assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1243   if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
1244     // This thread went through the OOM during evac protocol. It is safe to return
1245     // the forward pointer. It must not attempt to evacuate any other objects.
1246     return ShenandoahBarrierSet::resolve_forwarded(p);
1247   }
1248 
1249   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1250 
1251   ShenandoahHeapRegion* r = heap_region_containing(p);
1252   assert(!r->is_humongous(), "never evacuate humongous objects");
1253 
1254   ShenandoahAffiliation target_gen = r->affiliation();
1255   return try_evacuate_object(p, thread, r, target_gen);
1256 }
1257 
1258 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1259                                                ShenandoahAffiliation target_gen) {
1260   assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1261   assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1262   bool alloc_from_lab = true;
1263   HeapWord* copy = nullptr;
1264   size_t size = p->size();
1265 
1266 #ifdef ASSERT
1267   if (ShenandoahOOMDuringEvacALot &&
1268       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1269     copy = nullptr;
1270   } else {
1271 #endif
1272     if (UseTLAB) {
1273       copy = allocate_from_gclab(thread, size);
1274       if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
1275         // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve.  Try resetting
1276         // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
1277         // TODO: is this right? using PLAB::min_size() here for gc lab size?
1278         ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
1279         copy = allocate_from_gclab(thread, size);
1280         // If we still get nullptr, we'll try a shared allocation below.
1281       }
1282     }
1283 
1284     if (copy == nullptr) {
1285       // If we failed to allocate in LAB, we'll try a shared allocation.
1286       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1287       copy = allocate_memory(req);
1288       alloc_from_lab = false;
1289     }
1290 #ifdef ASSERT
1291   }
1292 #endif
1293 
1294   if (copy == nullptr) {
1295     control_thread()->handle_alloc_failure_evac(size);
1296 
1297     _oom_evac_handler.handle_out_of_memory_during_evacuation();
1298 
1299     return ShenandoahBarrierSet::resolve_forwarded(p);
1300   }
1301 
1302   // Copy the object:
1303   _evac_tracker->begin_evacuation(thread, size * HeapWordSize);
1304   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1305 
1306   oop copy_val = cast_to_oop(copy);
1307 
1308   // Try to install the new forwarding pointer.
1309   ContinuationGCSupport::relativize_stack_chunk(copy_val);
1310 
1311   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1312   if (result == copy_val) {
1313     // Successfully evacuated. Our copy is now the public one!
1314     _evac_tracker->end_evacuation(thread, size * HeapWordSize);
1315     shenandoah_assert_correct(nullptr, copy_val);
1316     return copy_val;
1317   }  else {
1318     // Failed to evacuate. We need to deal with the object that is left behind. Since this
1319     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1320     // But if it happens to contain references to evacuated regions, those references would
1321     // not get updated for this stale copy during this cycle, and we will crash while scanning
1322     // it the next cycle.
1323     if (alloc_from_lab) {
1324       // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1325       // object will overwrite this stale copy, or the filler object on LAB retirement will
1326       // do this.
1327       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
1328     } else {
1329       // For non-LAB allocations, we have no way to retract the allocation, and
1330       // have to explicitly overwrite the copy with the filler object. With that overwrite,
1331       // we have to keep the fwdptr initialized and pointing to our (stale) copy.
1332       assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
1333       fill_with_object(copy, size);
1334       shenandoah_assert_correct(nullptr, copy_val);
1335       // For non-LAB allocations, the object has already been registered
1336     }
1337     shenandoah_assert_correct(nullptr, result);
1338     return result;
1339   }
1340 }
1341 
1342 void ShenandoahHeap::trash_cset_regions() {
1343   ShenandoahHeapLocker locker(lock());
1344 
1345   ShenandoahCollectionSet* set = collection_set();
1346   ShenandoahHeapRegion* r;
1347   set->clear_current_index();
1348   while ((r = set->next()) != nullptr) {
1349     r->make_trash();
1350   }
1351   collection_set()->clear();
1352 }
1353 
1354 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1355   st->print_cr("Heap Regions:");
1356   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1357   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1358   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1359   st->print_cr("UWM=update watermark, U=used");
1360   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1361   st->print_cr("S=shared allocs, L=live data");
1362   st->print_cr("CP=critical pins");
1363 
1364   for (size_t i = 0; i < num_regions(); i++) {
1365     get_region(i)->print_on(st);
1366   }
1367 }
1368 
1369 size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1370   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1371 
1372   oop humongous_obj = cast_to_oop(start->bottom());
1373   size_t size = humongous_obj->size();
1374   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1375   size_t index = start->index() + required_regions - 1;
1376 
1377   assert(!start->has_live(), "liveness must be zero");
1378 
1379   for(size_t i = 0; i < required_regions; i++) {
1380     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1381     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1382     ShenandoahHeapRegion* region = get_region(index --);
1383 
1384     assert(region->is_humongous(), "expect correct humongous start or continuation");
1385     assert(!region->is_cset(), "Humongous region should not be in collection set");
1386 
1387     region->make_trash_immediate();
1388   }
1389   return required_regions;
1390 }
1391 
1392 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1393 public:
1394   ShenandoahCheckCleanGCLABClosure() {}
1395   void do_thread(Thread* thread) {
1396     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1397     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1398     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1399 
1400     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1401       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1402       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1403       assert(plab->words_remaining() == 0, "PLAB should not need retirement");
1404     }
1405   }
1406 };
1407 
1408 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1409 private:
1410   bool const _resize;
1411 public:
1412   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1413   void do_thread(Thread* thread) {
1414     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1415     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1416     gclab->retire();
1417     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1418       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1419     }
1420 
1421     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1422       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1423       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1424 
1425       // There are two reasons to retire all plabs between old-gen evacuation passes.
1426       //  1. We need to make the plab memory parsable by remembered-set scanning.
1427       //  2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
1428       ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
1429       if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
1430         ShenandoahThreadLocalData::set_plab_size(thread, 0);
1431       }
1432     }
1433   }
1434 };
1435 
1436 void ShenandoahHeap::labs_make_parsable() {
1437   assert(UseTLAB, "Only call with UseTLAB");
1438 
1439   ShenandoahRetireGCLABClosure cl(false);
1440 
1441   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1442     ThreadLocalAllocBuffer& tlab = t->tlab();
1443     tlab.make_parsable();
1444     cl.do_thread(t);
1445   }
1446 
1447   workers()->threads_do(&cl);
1448 }
1449 
1450 void ShenandoahHeap::tlabs_retire(bool resize) {
1451   assert(UseTLAB, "Only call with UseTLAB");
1452   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1514   }
1515   return nullptr;
1516 }
1517 
1518 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1519   ShenandoahHeapRegion* r = heap_region_containing(addr);
1520   return r->block_is_obj(addr);
1521 }
1522 
1523 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1524   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1525 }
1526 
1527 void ShenandoahHeap::prepare_for_verify() {
1528   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1529     labs_make_parsable();
1530   }
1531 }
1532 
1533 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1534   if (_shenandoah_policy->is_at_shutdown()) {
1535     return;
1536   }
1537 
1538   if (_control_thread != nullptr) {
1539     tcl->do_thread(_control_thread);
1540   }
1541 
1542   workers()->threads_do(tcl);
1543   if (_safepoint_workers != nullptr) {
1544     _safepoint_workers->threads_do(tcl);
1545   }
1546 }
1547 
1548 void ShenandoahHeap::print_tracing_info() const {
1549   LogTarget(Info, gc, stats) lt;
1550   if (lt.is_enabled()) {
1551     ResourceMark rm;
1552     LogStream ls(lt);
1553 
1554     phase_timings()->print_global_on(&ls);
1555 
1556     ls.cr();
1557     ls.cr();
1558 
1559     shenandoah_policy()->print_gc_stats(&ls);
1560 
1561     ls.cr();
1562 
1563     evac_tracker()->print_global_on(&ls);
1564 
1565     ls.cr();
1566     ls.cr();
1567   }
1568 }
1569 
1570 void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) {
1571   shenandoah_policy()->record_collection_cause(cause);
1572 
1573   set_gc_cause(cause);
1574   set_gc_generation(generation);
1575 
1576   generation->heuristics()->record_cycle_start();
1577 }
1578 
1579 void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) {
1580   generation->heuristics()->record_cycle_end();
1581   if (mode()->is_generational() && generation->is_global()) {
1582     // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well
1583     young_generation()->heuristics()->record_cycle_end();
1584     old_generation()->heuristics()->record_cycle_end();
1585   }
1586   set_gc_cause(GCCause::_no_gc);
1587 }
1588 
1589 void ShenandoahHeap::verify(VerifyOption vo) {
1590   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1591     if (ShenandoahVerify) {
1592       verifier()->verify_generic(vo);
1593     } else {
1594       // TODO: Consider allocating verification bitmaps on demand,
1595       // and turn this on unconditionally.
1596     }
1597   }
1598 }
1599 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1600   return _free_set->capacity();
1601 }
1602 
1603 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1604 private:
1605   MarkBitMap* _bitmap;
1606   ShenandoahScanObjectStack* _oop_stack;

1890       if (start >= max) break;
1891 
1892       for (size_t i = cur; i < end; i++) {
1893         ShenandoahHeapRegion* current = _heap->get_region(i);
1894         _blk->heap_region_do(current);
1895       }
1896     }
1897   }
1898 };
1899 
1900 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1901   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1902   if (num_regions() > ShenandoahParallelRegionStride) {
1903     ShenandoahParallelHeapRegionTask task(blk);
1904     workers()->run_task(&task);
1905   } else {
1906     heap_region_iterate(blk);
1907   }
1908 }
1909 























1910 class ShenandoahRendezvousClosure : public HandshakeClosure {
1911 public:
1912   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1913   inline void do_thread(Thread* thread) {}
1914 };
1915 
1916 void ShenandoahHeap::rendezvous_threads() {
1917   ShenandoahRendezvousClosure cl;
1918   Handshake::execute(&cl);
1919 }
1920 
1921 void ShenandoahHeap::recycle_trash() {
1922   free_set()->recycle_trash();
1923 }
1924 



































































































1925 void ShenandoahHeap::do_class_unloading() {
1926   _unloader.unload();
1927   if (mode()->is_generational()) {
1928     old_generation()->set_parseable(false);
1929   }
1930 }
1931 
1932 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1933   // Weak refs processing
1934   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1935                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1936   ShenandoahTimingsTracker t(phase);
1937   ShenandoahGCWorkerPhase worker_phase(phase);
1938   active_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
1939 }
1940 
1941 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1942   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1943 
1944   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1945   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1946   // for future GCLABs here.
1947   if (UseTLAB) {
1948     ShenandoahGCPhase phase(concurrent ?
1949                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1950                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1951     gclabs_retire(ResizeTLAB);
1952   }
1953 
1954   _update_refs_iterator.reset();
1955 }
1956 
1957 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1958   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1959   if (_gc_state_changed) {
1960     _gc_state_changed = false;
1961     char state = gc_state();
1962     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1963       ShenandoahThreadLocalData::set_gc_state(t, state);
1964     }
1965   }
1966 }
1967 
1968 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1969   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1970   _gc_state.set_cond(mask, value);
1971   _gc_state_changed = true;
1972 }
1973 
1974 void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
1975   uint mask;
1976   assert(!has_forwarded_objects(), "Young marking is not concurrent with evacuation");
1977   if (!in_progress && is_concurrent_old_mark_in_progress()) {
1978     assert(mode()->is_generational(), "Only generational GC has old marking");
1979     assert(_gc_state.is_set(MARKING), "concurrent_old_marking_in_progress implies MARKING");
1980     // If old-marking is in progress when we turn off YOUNG_MARKING, leave MARKING (and OLD_MARKING) on
1981     mask = YOUNG_MARKING;
1982   } else {
1983     mask = MARKING | YOUNG_MARKING;
1984   }
1985   set_gc_state(mask, in_progress);
1986   manage_satb_barrier(in_progress);
1987 }
1988 
1989 void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
1990 #ifdef ASSERT
1991   // has_forwarded_objects() iff UPDATEREFS or EVACUATION
1992   bool has_forwarded = has_forwarded_objects();
1993   bool updating_or_evacuating = _gc_state.is_set(UPDATEREFS | EVACUATION);
1994   bool evacuating = _gc_state.is_set(EVACUATION);
1995   assert ((has_forwarded == updating_or_evacuating) || (evacuating && !has_forwarded && collection_set()->is_empty()),
1996           "Updating or evacuating iff has forwarded objects, or if evacuation phase is promoting in place without forwarding");
1997 #endif
1998   if (!in_progress && is_concurrent_young_mark_in_progress()) {
1999     // If young-marking is in progress when we turn off OLD_MARKING, leave MARKING (and YOUNG_MARKING) on
2000     assert(_gc_state.is_set(MARKING), "concurrent_young_marking_in_progress implies MARKING");
2001     set_gc_state(OLD_MARKING, in_progress);
2002   } else {
2003     set_gc_state(MARKING | OLD_MARKING, in_progress);
2004   }
2005   manage_satb_barrier(in_progress);
2006 }
2007 
2008 bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const {
2009   return old_generation()->is_preparing_for_mark();
2010 }
2011 
2012 void ShenandoahHeap::manage_satb_barrier(bool active) {
2013   if (is_concurrent_mark_in_progress()) {
2014     // Ignore request to deactivate barrier while concurrent mark is in progress.
2015     // Do not attempt to re-activate the barrier if it is already active.
2016     if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
2017       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2018     }
2019   } else {
2020     // No concurrent marking is in progress so honor request to deactivate,
2021     // but only if the barrier is already active.
2022     if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
2023       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2024     }
2025   }
2026 }
2027 
2028 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2029   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2030   set_gc_state(EVACUATION, in_progress);
2031 }
2032 
2033 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2034   if (in_progress) {
2035     _concurrent_strong_root_in_progress.set();
2036   } else {
2037     _concurrent_strong_root_in_progress.unset();
2038   }
2039 }
2040 
2041 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
2042   set_gc_state(WEAK_ROOTS, cond);
2043 }
2044 
2045 GCTracer* ShenandoahHeap::tracer() {
2046   return shenandoah_policy()->tracer();
2047 }
2048 
2049 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2050   return _free_set->used();
2051 }
2052 
2053 bool ShenandoahHeap::try_cancel_gc() {
2054   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2055   return prev == CANCELLABLE;
2056 }
2057 
2058 void ShenandoahHeap::cancel_concurrent_mark() {
2059   _young_generation->cancel_marking();
2060   _old_generation->cancel_marking();
2061   _global_generation->cancel_marking();
2062 
2063   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
2064 }
2065 
2066 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2067   if (try_cancel_gc()) {
2068     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2069     log_info(gc)("%s", msg.buffer());
2070     Events::log(Thread::current(), "%s", msg.buffer());
2071     _cancel_requested_time = os::elapsedTime();
2072   }
2073 }
2074 
2075 uint ShenandoahHeap::max_workers() {
2076   return _max_workers;
2077 }
2078 
2079 void ShenandoahHeap::stop() {
2080   // The shutdown sequence should be able to terminate when GC is running.
2081 
2082   // Step 0. Notify policy to disable event recording.
2083   _shenandoah_policy->record_shutdown();
2084 
2085   // Step 1. Notify control thread that we are in shutdown.
2086   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2087   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2088   control_thread()->prepare_for_graceful_shutdown();
2089 
2090   // Step 2. Notify GC workers that we are cancelling GC.
2091   cancel_gc(GCCause::_shenandoah_stop_vm);

2175 }
2176 
2177 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2178   set_gc_state(HAS_FORWARDED, cond);
2179 }
2180 
2181 void ShenandoahHeap::set_unload_classes(bool uc) {
2182   _unload_classes.set_cond(uc);
2183 }
2184 
2185 bool ShenandoahHeap::unload_classes() const {
2186   return _unload_classes.is_set();
2187 }
2188 
2189 address ShenandoahHeap::in_cset_fast_test_addr() {
2190   ShenandoahHeap* heap = ShenandoahHeap::heap();
2191   assert(heap->collection_set() != nullptr, "Sanity");
2192   return (address) heap->collection_set()->biased_map_address();
2193 }
2194 




2195 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2196   if (mode()->is_generational()) {
2197     young_generation()->reset_bytes_allocated_since_gc_start();
2198     old_generation()->reset_bytes_allocated_since_gc_start();
2199   }
2200 
2201   global_generation()->reset_bytes_allocated_since_gc_start();
2202 }
2203 
2204 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2205   _degenerated_gc_in_progress.set_cond(in_progress);
2206 }
2207 
2208 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2209   _full_gc_in_progress.set_cond(in_progress);
2210 }
2211 
2212 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2213   assert (is_full_gc_in_progress(), "should be");
2214   _full_gc_move_in_progress.set_cond(in_progress);
2215 }
2216 
2217 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2218   set_gc_state(UPDATEREFS, in_progress);
2219 }
2220 
2221 void ShenandoahHeap::register_nmethod(nmethod* nm) {

2245     if (r->is_active()) {
2246       if (r->is_pinned()) {
2247         if (r->pin_count() == 0) {
2248           r->make_unpinned();
2249         }
2250       } else {
2251         if (r->pin_count() > 0) {
2252           r->make_pinned();
2253         }
2254       }
2255     }
2256   }
2257 
2258   assert_pinned_region_status();
2259 }
2260 
2261 #ifdef ASSERT
2262 void ShenandoahHeap::assert_pinned_region_status() {
2263   for (size_t i = 0; i < num_regions(); i++) {
2264     ShenandoahHeapRegion* r = get_region(i);
2265     if (active_generation()->contains(r)) {
2266       assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2267              "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2268     }
2269   }
2270 }
2271 #endif
2272 
2273 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2274   return _gc_timer;
2275 }
2276 
2277 void ShenandoahHeap::prepare_concurrent_roots() {
2278   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2279   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2280   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2281   set_concurrent_weak_root_in_progress(true);
2282   if (unload_classes()) {
2283     _unloader.prepare();
2284   }
2285 }
2286 
2287 void ShenandoahHeap::finish_concurrent_roots() {
2288   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2289   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2290   if (unload_classes()) {
2291     _unloader.finish();
2292   }
2293 }
2294 
2295 #ifdef ASSERT
2296 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2297   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2298 
2299   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2300     // Use ParallelGCThreads inside safepoints
2301     assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads (%u) within safepoint, not %u",
2302            ParallelGCThreads, nworkers);



2303   } else {
2304     // Use ConcGCThreads outside safepoints
2305     assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2306            ConcGCThreads, nworkers);



2307   }
2308 }
2309 #endif
2310 
2311 ShenandoahVerifier* ShenandoahHeap::verifier() {
2312   guarantee(ShenandoahVerify, "Should be enabled");
2313   assert (_verifier != nullptr, "sanity");
2314   return _verifier;
2315 }
2316 
2317 template<bool CONCURRENT>
2318 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2319 private:
2320   ShenandoahHeap* _heap;
2321   ShenandoahRegionIterator* _regions;
2322 public:
2323   explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2324     WorkerTask("Shenandoah Update References"),
2325     _heap(ShenandoahHeap::heap()),
2326     _regions(regions) {
2327   }
2328 
2329   void work(uint worker_id) {
2330     if (CONCURRENT) {
2331       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2332       ShenandoahSuspendibleThreadSetJoiner stsj;
2333       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
2334     } else {
2335       ShenandoahParallelWorkerSession worker_session(worker_id);
2336       do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
2337     }
2338   }
2339 
2340 private:
2341   template<class T>
2342   void do_work(uint worker_id) {
2343     if (CONCURRENT && (worker_id == 0)) {
2344       // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
2345       // results of evacuation.  These reserves are no longer necessary because evacuation has completed.
2346       size_t cset_regions = _heap->collection_set()->count();
2347       // We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled, because
2348       // we need the reclaimed collection set regions to replenish the collector reserves
2349       _heap->free_set()->move_collector_sets_to_mutator(cset_regions);
2350     }
2351     // If !CONCURRENT, there's no value in expanding Mutator free set
2352     T cl;
2353     ShenandoahHeapRegion* r = _regions->next();

2354     while (r != nullptr) {
2355       HeapWord* update_watermark = r->get_update_watermark();
2356       assert (update_watermark >= r->bottom(), "sanity");
2357       if (r->is_active() && !r->is_cset()) {
2358         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2359         if (ShenandoahPacing) {
2360           _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2361         }
2362       }
2363       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2364         return;
2365       }
2366       r = _regions->next();
2367     }
2368   }
2369 };
2370 
2371 void ShenandoahHeap::update_heap_references(bool concurrent) {
2372   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2373 
2374   if (concurrent) {
2375     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2376     workers()->run_task(&task);
2377   } else {
2378     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2379     workers()->run_task(&task);
2380   }
2381 }
2382 

2383 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2384 private:
2385   ShenandoahMarkingContext* _ctx;
2386   ShenandoahHeapLock* const _lock;
2387   bool _is_generational;
2388 
2389 public:
2390   ShenandoahFinalUpdateRefsUpdateRegionStateClosure(ShenandoahMarkingContext* ctx) :
2391     _ctx(ctx), _lock(ShenandoahHeap::heap()->lock()),
2392     _is_generational(ShenandoahHeap::heap()->mode()->is_generational()) { }
2393 
2394   void heap_region_do(ShenandoahHeapRegion* r) override {
2395 
2396     // Maintenance of region age must follow evacuation in order to account for evacuation allocations within survivor
2397     // regions.  We consult region age during the subsequent evacuation to determine whether certain objects need to
2398     // be promoted.
2399     if (_is_generational && r->is_young() && r->is_active()) {
2400       HeapWord *tams = _ctx->top_at_mark_start(r);
2401       HeapWord *top = r->top();
2402 
2403       // Allocations move the watermark when top moves.  However, compacting
2404       // objects will sometimes lower top beneath the watermark, after which,
2405       // attempts to read the watermark will assert out (watermark should not be
2406       // higher than top).
2407       if (top > tams) {
2408         // There have been allocations in this region since the start of the cycle.
2409         // Any objects new to this region must not assimilate elevated age.
2410         r->reset_age();
2411       } else if (ShenandoahGenerationalHeap::heap()->is_aging_cycle()) {
2412         r->increment_age();
2413       }
2414     }
2415 

2416     // Drop unnecessary "pinned" state from regions that does not have CP marks
2417     // anymore, as this would allow trashing them.

2418     if (r->is_active()) {
2419       if (r->is_pinned()) {
2420         if (r->pin_count() == 0) {
2421           ShenandoahHeapLocker locker(_lock);
2422           r->make_unpinned();
2423         }
2424       } else {
2425         if (r->pin_count() > 0) {
2426           ShenandoahHeapLocker locker(_lock);
2427           r->make_pinned();
2428         }
2429       }
2430     }
2431   }
2432 
2433   bool is_thread_safe() override { return true; }
2434 };
2435 
2436 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2437   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2438   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2439 
2440   {
2441     ShenandoahGCPhase phase(concurrent ?
2442                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2443                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2444     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl (active_generation()->complete_marking_context());
2445     parallel_heap_region_iterate(&cl);
2446 
2447     assert_pinned_region_status();
2448   }
2449 
2450   {
2451     ShenandoahGCPhase phase(concurrent ?
2452                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2453                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2454     trash_cset_regions();
2455   }
2456 }
2457 
2458 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2459   ShenandoahGCPhase phase(concurrent ?
2460                           ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2461                           ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2462   ShenandoahHeapLocker locker(lock());
2463   size_t young_cset_regions, old_cset_regions;
2464   size_t first_old_region, last_old_region, old_region_count;
2465   _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
2466   // If there are no old regions, first_old_region will be greater than last_old_region
2467   assert((first_old_region > last_old_region) ||
2468          ((last_old_region + 1 - first_old_region >= old_region_count) &&
2469           get_region(first_old_region)->is_old() && get_region(last_old_region)->is_old()),
2470          "sanity: old_region_count: " SIZE_FORMAT ", first_old_region: " SIZE_FORMAT ", last_old_region: " SIZE_FORMAT,
2471          old_region_count, first_old_region, last_old_region);
2472 
2473   if (mode()->is_generational()) {
2474 #ifdef ASSERT
2475     if (ShenandoahVerify) {
2476       verifier()->verify_before_rebuilding_free_set();
2477     }
2478 #endif
2479 
2480     // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
2481     // available for transfer to old. Note that transfer of humongous regions does not impact available.
2482     size_t allocation_runway = young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
2483     ShenandoahGenerationalHeap::heap()->compute_old_generation_balance(allocation_runway, old_cset_regions);
2484 
2485     // Total old_available may have been expanded to hold anticipated promotions.  We trigger if the fragmented available
2486     // memory represents more than 16 regions worth of data.  Note that fragmentation may increase when we promote regular
2487     // regions in place when many of these regular regions have an abundant amount of available memory within them.  Fragmentation
2488     // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
2489     //
2490     // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
2491     // within partially consumed regions of memory.
2492   }
2493   // Rebuild free set based on adjusted generation sizes.
2494   _free_set->rebuild(young_cset_regions, old_cset_regions);
2495 
2496   if (mode()->is_generational()) {
2497     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2498     ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
2499     ShenandoahOldHeuristics* old_heuristics = old_gen->heuristics();
2500     old_heuristics->trigger_maybe(first_old_region, last_old_region, old_region_count, num_regions());
2501   }
2502 }
2503 
2504 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2505   print_on(st);
2506   st->cr();
2507   print_heap_regions_on(st);
2508 }
2509 
2510 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2511   size_t slice = r->index() / _bitmap_regions_per_slice;
2512 
2513   size_t regions_from = _bitmap_regions_per_slice * slice;
2514   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2515   for (size_t g = regions_from; g < regions_to; g++) {
2516     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2517     if (skip_self && g == r->index()) continue;
2518     if (get_region(g)->is_committed()) {
2519       return true;
2520     }

2568   }
2569 
2570   // Uncommit the bitmap slice:
2571   size_t slice = r->index() / _bitmap_regions_per_slice;
2572   size_t off = _bitmap_bytes_per_slice * slice;
2573   size_t len = _bitmap_bytes_per_slice;
2574   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2575     return false;
2576   }
2577   return true;
2578 }
2579 
2580 void ShenandoahHeap::safepoint_synchronize_begin() {
2581   SuspendibleThreadSet::synchronize();
2582 }
2583 
2584 void ShenandoahHeap::safepoint_synchronize_end() {
2585   SuspendibleThreadSet::desynchronize();
2586 }
2587 








2588 void ShenandoahHeap::try_inject_alloc_failure() {
2589   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2590     _inject_alloc_failure.set();
2591     os::naked_short_sleep(1);
2592     if (cancelled_gc()) {
2593       log_info(gc)("Allocation failure was successfully injected");
2594     }
2595   }
2596 }
2597 
2598 bool ShenandoahHeap::should_inject_alloc_failure() {
2599   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2600 }
2601 
2602 void ShenandoahHeap::initialize_serviceability() {
2603   _memory_pool = new ShenandoahMemoryPool(this);
2604   _cycle_memory_manager.add_pool(_memory_pool);
2605   _stw_memory_manager.add_pool(_memory_pool);
2606 }
2607 
2608 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2609   GrowableArray<GCMemoryManager*> memory_managers(2);
2610   memory_managers.append(&_cycle_memory_manager);
2611   memory_managers.append(&_stw_memory_manager);
2612   return memory_managers;
2613 }
2614 
2615 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2616   GrowableArray<MemoryPool*> memory_pools(1);
2617   memory_pools.append(_memory_pool);
2618   return memory_pools;
2619 }
2620 
2621 MemoryUsage ShenandoahHeap::memory_usage() {
2622   return MemoryUsage(_initial_size, used(), committed(), max_capacity());
2623 }
2624 
2625 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2626   _heap(ShenandoahHeap::heap()),
2627   _index(0) {}
2628 
2629 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2630   _heap(heap),
2631   _index(0) {}
2632 
2633 void ShenandoahRegionIterator::reset() {
2634   _index = 0;
2635 }
2636 
2637 bool ShenandoahRegionIterator::has_next() const {
2638   return _index < _heap->num_regions();
2639 }
2640 
2641 char ShenandoahHeap::gc_state() const {
2642   return _gc_state.raw_value();

2667   }
2668 }
2669 
2670 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2671   if (is_idle()) return false;
2672 
2673   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2674   // marking phase.
2675   if (is_concurrent_mark_in_progress() &&
2676      !marking_context()->allocated_after_mark_start(obj)) {
2677     return true;
2678   }
2679 
2680   // Can not guarantee obj is deeply good.
2681   if (has_forwarded_objects()) {
2682     return true;
2683   }
2684 
2685   return false;
2686 }
2687 
2688 ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const {
2689   if (!mode()->is_generational()) {
2690     return global_generation();
2691   } else if (affiliation == YOUNG_GENERATION) {
2692     return young_generation();
2693   } else if (affiliation == OLD_GENERATION) {
2694     return old_generation();
2695   }
2696 
2697   ShouldNotReachHere();
2698   return nullptr;
2699 }
2700 
2701 void ShenandoahHeap::log_heap_status(const char* msg) const {
2702   if (mode()->is_generational()) {
2703     young_generation()->log_status(msg);
2704     old_generation()->log_status(msg);
2705   } else {
2706     global_generation()->log_status(msg);
2707   }
2708 }
2709 
2710 void ShenandoahHeap::clear_cards_for(ShenandoahHeapRegion* region) {
2711   if (mode()->is_generational()) {
2712     _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom()));
2713   }
2714 }
2715 
2716 void ShenandoahHeap::mark_card_as_dirty(void* location) {
2717   if (mode()->is_generational()) {
2718     _card_scan->mark_card_as_dirty((HeapWord*)location);
2719   }
2720 }
< prev index next >