< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page

   1 /*
   2  * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.

   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "memory/allocation.hpp"
  28 #include "memory/universe.hpp"
  29 
  30 #include "gc/shared/classUnloadingContext.hpp"
  31 #include "gc/shared/gcArguments.hpp"
  32 #include "gc/shared/gcTimer.hpp"
  33 #include "gc/shared/gcTraceTime.inline.hpp"
  34 #include "gc/shared/locationPrinter.inline.hpp"
  35 #include "gc/shared/memAllocator.hpp"
  36 #include "gc/shared/plab.hpp"
  37 #include "gc/shared/tlab_globals.hpp"
  38 



  39 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  40 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  41 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  42 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  43 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  45 #include "gc/shenandoah/shenandoahControlThread.hpp"
  46 #include "gc/shenandoah/shenandoahFreeSet.hpp"



  47 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  48 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  49 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  50 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  51 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  52 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  53 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  54 #include "gc/shenandoah/shenandoahMetrics.hpp"
  55 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"

  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"

  62 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  63 #include "gc/shenandoah/shenandoahUtils.hpp"
  64 #include "gc/shenandoah/shenandoahVerifier.hpp"
  65 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  66 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  67 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  68 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"


  69 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  70 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  71 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"


  72 #if INCLUDE_JFR
  73 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  74 #endif
  75 
  76 #include "classfile/systemDictionary.hpp"
  77 #include "code/codeCache.hpp"
  78 #include "memory/classLoaderMetaspace.hpp"
  79 #include "memory/metaspaceUtils.hpp"
  80 #include "nmt/mallocTracker.hpp"
  81 #include "nmt/memTracker.hpp"
  82 #include "oops/compressedOops.inline.hpp"
  83 #include "prims/jvmtiTagMap.hpp"
  84 #include "runtime/atomic.hpp"
  85 #include "runtime/globals.hpp"
  86 #include "runtime/interfaceSupport.inline.hpp"
  87 #include "runtime/java.hpp"
  88 #include "runtime/orderAccess.hpp"
  89 #include "runtime/safepointMechanism.hpp"
  90 #include "runtime/stackWatermarkSet.hpp"
  91 #include "runtime/vmThread.hpp"

 144 jint ShenandoahHeap::initialize() {
 145   //
 146   // Figure out heap sizing
 147   //
 148 
 149   size_t init_byte_size = InitialHeapSize;
 150   size_t min_byte_size  = MinHeapSize;
 151   size_t max_byte_size  = MaxHeapSize;
 152   size_t heap_alignment = HeapAlignment;
 153 
 154   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 155 
 156   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 157   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 158 
 159   _num_regions = ShenandoahHeapRegion::region_count();
 160   assert(_num_regions == (max_byte_size / reg_size_bytes),
 161          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 162          _num_regions, max_byte_size, reg_size_bytes);
 163 
 164   // Now we know the number of regions, initialize the heuristics.
 165   initialize_heuristics();
 166 
 167   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 168   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 169   assert(num_committed_regions <= _num_regions, "sanity");
 170   _initial_size = num_committed_regions * reg_size_bytes;
 171 
 172   size_t num_min_regions = min_byte_size / reg_size_bytes;
 173   num_min_regions = MIN2(num_min_regions, _num_regions);
 174   assert(num_min_regions <= _num_regions, "sanity");
 175   _minimum_size = num_min_regions * reg_size_bytes;
 176 
 177   // Default to max heap size.
 178   _soft_max_size = _num_regions * reg_size_bytes;
 179 
 180   _committed = _initial_size;
 181 
 182   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 183   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 184   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 185 
 186   //

 200                                           heap_rs.size(), heap_rs.page_size());
 201 
 202 #if SHENANDOAH_OPTIMIZED_MARKTASK
 203   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 204   // Fail if we ever attempt to address more than we can.
 205   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 206     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 207                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 208                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 209                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 210     vm_exit_during_initialization("Fatal Error", buf);
 211   }
 212 #endif
 213 
 214   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 215   if (!_heap_region_special) {
 216     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 217                               "Cannot commit heap memory");
 218   }
 219 






















 220   //
 221   // Reserve and commit memory for bitmap(s)
 222   //
 223 
 224   size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 225   _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
 226 
 227   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 228 
 229   guarantee(bitmap_bytes_per_region != 0,
 230             "Bitmap bytes per region should not be zero");
 231   guarantee(is_power_of_2(bitmap_bytes_per_region),
 232             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 233 
 234   if (bitmap_page_size > bitmap_bytes_per_region) {
 235     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 236     _bitmap_bytes_per_slice = bitmap_page_size;
 237   } else {
 238     _bitmap_regions_per_slice = 1;
 239     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 240   }
 241 
 242   guarantee(_bitmap_regions_per_slice >= 1,
 243             "Should have at least one region per slice: " SIZE_FORMAT,
 244             _bitmap_regions_per_slice);
 245 
 246   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 247             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 248             _bitmap_bytes_per_slice, bitmap_page_size);
 249 
 250   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 251   os::trace_page_sizes_for_requested_size("Mark Bitmap",
 252                                           bitmap_size_orig, bitmap_page_size,
 253                                           bitmap.base(),
 254                                           bitmap.size(), bitmap.page_size());
 255   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 256   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 257   _bitmap_region_special = bitmap.special();
 258 
 259   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 260                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 261   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 262   if (!_bitmap_region_special) {
 263     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 264                               "Cannot commit bitmap memory");
 265   }
 266 
 267   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
 268 
 269   if (ShenandoahVerify) {
 270     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 271     os::trace_page_sizes_for_requested_size("Verify Bitmap",
 272                                             bitmap_size_orig, bitmap_page_size,
 273                                             verify_bitmap.base(),
 274                                             verify_bitmap.size(), verify_bitmap.page_size());
 275     if (!verify_bitmap.special()) {
 276       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 277                                 "Cannot commit verification bitmap memory");
 278     }
 279     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 280     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 281     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 282     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 283   }
 284 
 285   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 286   size_t aux_bitmap_page_size = bitmap_page_size;
 287 #ifdef LINUX

 338       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 339       cset_rs = ReservedSpace(cset_size, cset_align, cset_page_size, req_addr);
 340       if (cset_rs.is_reserved()) {
 341         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 342         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 343         break;
 344       }
 345     }
 346 
 347     if (_collection_set == nullptr) {
 348       cset_rs = ReservedSpace(cset_size, cset_align, os::vm_page_size());
 349       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 350     }
 351     os::trace_page_sizes_for_requested_size("Collection Set",
 352                                             cset_size, cset_page_size,
 353                                             cset_rs.base(),
 354                                             cset_rs.size(), cset_rs.page_size());
 355   }
 356 
 357   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);

 358   _free_set = new ShenandoahFreeSet(this, _num_regions);
 359 
 360   {
 361     ShenandoahHeapLocker locker(lock());
 362 

 363     for (size_t i = 0; i < _num_regions; i++) {
 364       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 365       bool is_committed = i < num_committed_regions;
 366       void* loc = region_storage.base() + i * region_align;
 367 
 368       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 369       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 370 
 371       _marking_context->initialize_top_at_mark_start(r);
 372       _regions[i] = r;
 373       assert(!collection_set()->is_in(i), "New region should not be in collection set");


 374     }
 375 
 376     // Initialize to complete
 377     _marking_context->mark_complete();

 378 
 379     _free_set->rebuild();



 380   }
 381 
 382   if (AlwaysPreTouch) {
 383     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 384     // before initialize() below zeroes it with initializing thread. For any given region,
 385     // we touch the region and the corresponding bitmaps from the same thread.
 386     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 387 
 388     _pretouch_heap_page_size = heap_page_size;
 389     _pretouch_bitmap_page_size = bitmap_page_size;
 390 
 391 #ifdef LINUX
 392     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 393     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 394     // them into huge one. Therefore, we need to pretouch with smaller pages.
 395     if (UseTransparentHugePages) {
 396       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 397       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 398     }
 399 #endif

 418     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 419   }
 420 
 421   // There should probably be Shenandoah-specific options for these,
 422   // just as there are G1-specific options.
 423   {
 424     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 425     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 426     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 427   }
 428 
 429   _monitoring_support = new ShenandoahMonitoringSupport(this);
 430   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 431   ShenandoahCodeRoots::initialize();
 432 
 433   if (ShenandoahPacing) {
 434     _pacer = new ShenandoahPacer(this);
 435     _pacer->setup_for_idle();
 436   }
 437 
 438   _control_thread = new ShenandoahControlThread();
 439 
 440   ShenandoahInitLogger::print();
 441 
 442   return JNI_OK;
 443 }
 444 








 445 void ShenandoahHeap::initialize_mode() {
 446   if (ShenandoahGCMode != nullptr) {
 447     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 448       _gc_mode = new ShenandoahSATBMode();
 449     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 450       _gc_mode = new ShenandoahIUMode();
 451     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 452       _gc_mode = new ShenandoahPassiveMode();


 453     } else {
 454       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 455     }
 456   } else {
 457     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 458   }
 459   _gc_mode->initialize_flags();
 460   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 461     vm_exit_during_initialization(
 462             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 463                     _gc_mode->name()));
 464   }
 465   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 466     vm_exit_during_initialization(
 467             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 468                     _gc_mode->name()));
 469   }
 470 }
 471 
 472 void ShenandoahHeap::initialize_heuristics() {
 473   assert(_gc_mode != nullptr, "Must be initialized");
 474   _heuristics = _gc_mode->initialize_heuristics();
 475 
 476   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 477     vm_exit_during_initialization(
 478             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 479                     _heuristics->name()));
 480   }
 481   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 482     vm_exit_during_initialization(
 483             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 484                     _heuristics->name()));
 485   }
 486 }
 487 
 488 #ifdef _MSC_VER
 489 #pragma warning( push )
 490 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 491 #endif
 492 
 493 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 494   CollectedHeap(),

 495   _initial_size(0),
 496   _used(0),
 497   _committed(0),
 498   _bytes_allocated_since_gc_start(0),
 499   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 500   _workers(nullptr),
 501   _safepoint_workers(nullptr),
 502   _heap_region_special(false),
 503   _num_regions(0),
 504   _regions(nullptr),
 505   _update_refs_iterator(this),
 506   _gc_state_changed(false),
 507   _gc_no_progress_count(0),



 508   _control_thread(nullptr),


 509   _shenandoah_policy(policy),
 510   _gc_mode(nullptr),
 511   _heuristics(nullptr),
 512   _free_set(nullptr),
 513   _pacer(nullptr),
 514   _verifier(nullptr),
 515   _phase_timings(nullptr),


 516   _monitoring_support(nullptr),
 517   _memory_pool(nullptr),
 518   _stw_memory_manager("Shenandoah Pauses"),
 519   _cycle_memory_manager("Shenandoah Cycles"),
 520   _gc_timer(new ConcurrentGCTimer()),
 521   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 522   _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
 523   _marking_context(nullptr),
 524   _bitmap_size(0),
 525   _bitmap_regions_per_slice(0),
 526   _bitmap_bytes_per_slice(0),
 527   _bitmap_region_special(false),
 528   _aux_bitmap_region_special(false),
 529   _liveness_cache(nullptr),
 530   _collection_set(nullptr)
 531 {
 532   // Initialize GC mode early, so we can adjust barrier support
 533   initialize_mode();
 534   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 535 
 536   _max_workers = MAX2(_max_workers, 1U);
 537   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 538   if (_workers == nullptr) {
 539     vm_exit_during_initialization("Failed necessary allocation.");
 540   } else {
 541     _workers->initialize_workers();
 542   }
 543 
 544   if (ParallelGCThreads > 1) {
 545     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
 546                                                 ParallelGCThreads);
 547     _safepoint_workers->initialize_workers();
 548   }
 549 }
 550 
 551 #ifdef _MSC_VER
 552 #pragma warning( pop )
 553 #endif
 554 
 555 class ShenandoahResetBitmapTask : public WorkerTask {
 556 private:
 557   ShenandoahRegionIterator _regions;
 558 
 559 public:
 560   ShenandoahResetBitmapTask() :
 561     WorkerTask("Shenandoah Reset Bitmap") {}
 562 
 563   void work(uint worker_id) {
 564     ShenandoahHeapRegion* region = _regions.next();
 565     ShenandoahHeap* heap = ShenandoahHeap::heap();
 566     ShenandoahMarkingContext* const ctx = heap->marking_context();
 567     while (region != nullptr) {
 568       if (heap->is_bitmap_slice_committed(region)) {
 569         ctx->clear_bitmap(region);
 570       }
 571       region = _regions.next();
 572     }
 573   }
 574 };
 575 
 576 void ShenandoahHeap::reset_mark_bitmap() {
 577   assert_gc_workers(_workers->active_workers());
 578   mark_incomplete_marking_context();
 579 
 580   ShenandoahResetBitmapTask task;
 581   _workers->run_task(&task);
 582 }
 583 
 584 void ShenandoahHeap::print_on(outputStream* st) const {
 585   st->print_cr("Shenandoah Heap");
 586   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 587                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 588                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 589                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 590                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 591   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 592                num_regions(),
 593                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 594                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 595 
 596   st->print("Status: ");
 597   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 598   if (is_concurrent_mark_in_progress())        st->print("marking, ");

 599   if (is_evacuation_in_progress())             st->print("evacuating, ");
 600   if (is_update_refs_in_progress())            st->print("updating refs, ");
 601   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 602   if (is_full_gc_in_progress())                st->print("full gc, ");
 603   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 604   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 605   if (is_concurrent_strong_root_in_progress() &&
 606       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 607 
 608   if (cancelled_gc()) {
 609     st->print("cancelled");
 610   } else {
 611     st->print("not cancelled");
 612   }
 613   st->cr();
 614 
 615   st->print_cr("Reserved region:");
 616   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 617                p2i(reserved_region().start()),
 618                p2i(reserved_region().end()));

 629   st->cr();
 630   MetaspaceUtils::print_on(st);
 631 
 632   if (Verbose) {
 633     st->cr();
 634     print_heap_regions_on(st);
 635   }
 636 }
 637 
 638 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 639 public:
 640   void do_thread(Thread* thread) {
 641     assert(thread != nullptr, "Sanity");
 642     assert(thread->is_Worker_thread(), "Only worker thread expected");
 643     ShenandoahThreadLocalData::initialize_gclab(thread);
 644   }
 645 };
 646 
 647 void ShenandoahHeap::post_initialize() {
 648   CollectedHeap::post_initialize();


 649   MutexLocker ml(Threads_lock);
 650 
 651   ShenandoahInitWorkerGCLABClosure init_gclabs;
 652   _workers->threads_do(&init_gclabs);
 653 
 654   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 655   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 656   _workers->set_initialize_gclab();
 657   if (_safepoint_workers != nullptr) {
 658     _safepoint_workers->threads_do(&init_gclabs);
 659     _safepoint_workers->set_initialize_gclab();
 660   }
 661 
 662   _heuristics->initialize();
 663 
 664   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
 665 }
 666 




 667 size_t ShenandoahHeap::used() const {
 668   return Atomic::load(&_used);
 669 }
 670 
 671 size_t ShenandoahHeap::committed() const {
 672   return Atomic::load(&_committed);
 673 }
 674 
 675 size_t ShenandoahHeap::available() const {
 676   return free_set()->available();
 677 }
 678 
 679 void ShenandoahHeap::increase_committed(size_t bytes) {
 680   shenandoah_assert_heaplocked_or_safepoint();
 681   _committed += bytes;
 682 }
 683 
 684 void ShenandoahHeap::decrease_committed(size_t bytes) {
 685   shenandoah_assert_heaplocked_or_safepoint();
 686   _committed -= bytes;
 687 }
 688 
 689 void ShenandoahHeap::increase_used(size_t bytes) {
 690   Atomic::add(&_used, bytes, memory_order_relaxed);









































 691 }
 692 
 693 void ShenandoahHeap::set_used(size_t bytes) {
 694   Atomic::store(&_used, bytes);



 695 }
 696 
 697 void ShenandoahHeap::decrease_used(size_t bytes) {
 698   assert(used() >= bytes, "never decrease heap size by more than we've left");
 699   Atomic::sub(&_used, bytes, memory_order_relaxed);


 700 }
 701 
 702 void ShenandoahHeap::increase_allocated(size_t bytes) {
 703   Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);



 704 }
 705 
 706 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 707   size_t bytes = words * HeapWordSize;
 708   if (!waste) {
 709     increase_used(bytes);
 710   }
 711   increase_allocated(bytes);


 712   if (ShenandoahPacing) {
 713     control_thread()->pacing_notify_alloc(words);
 714     if (waste) {
 715       pacer()->claim_for_alloc(words, true);
 716     }
 717   }
 718 }
 719 
 720 size_t ShenandoahHeap::capacity() const {
 721   return committed();
 722 }
 723 
 724 size_t ShenandoahHeap::max_capacity() const {
 725   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 726 }
 727 
 728 size_t ShenandoahHeap::soft_max_capacity() const {
 729   size_t v = Atomic::load(&_soft_max_size);
 730   assert(min_capacity() <= v && v <= max_capacity(),
 731          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 732          min_capacity(), v, max_capacity());
 733   return v;
 734 }
 735 
 736 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 737   assert(min_capacity() <= v && v <= max_capacity(),
 738          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 739          min_capacity(), v, max_capacity());
 740   Atomic::store(&_soft_max_size, v);
 741 }
 742 
 743 size_t ShenandoahHeap::min_capacity() const {
 744   return _minimum_size;
 745 }
 746 
 747 size_t ShenandoahHeap::initial_capacity() const {
 748   return _initial_size;
 749 }
 750 
 751 bool ShenandoahHeap::is_in(const void* p) const {
 752   HeapWord* heap_base = (HeapWord*) base();
 753   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 754   return p >= heap_base && p < last_region_end;
 755 }
 756 
 757 void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
 758   assert (ShenandoahUncommit, "should be enabled");
 759 
 760   // Determine if there is work to do. This avoids taking heap lock if there is
 761   // no work available, avoids spamming logs with superfluous logging messages,
 762   // and minimises the amount of work while locks are taken.
 763 
 764   if (committed() <= shrink_until) return;
 765 
 766   bool has_work = false;
 767   for (size_t i = 0; i < num_regions(); i++) {
 768     ShenandoahHeapRegion* r = get_region(i);
 769     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 770       has_work = true;
 771       break;
 772     }
 773   }
 774 
 775   if (has_work) {
 776     static const char* msg = "Concurrent uncommit";

 816   size_t old_soft_max = soft_max_capacity();
 817   if (new_soft_max != old_soft_max) {
 818     new_soft_max = MAX2(min_capacity(), new_soft_max);
 819     new_soft_max = MIN2(max_capacity(), new_soft_max);
 820     if (new_soft_max != old_soft_max) {
 821       log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
 822                    byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
 823                    byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
 824       );
 825       set_soft_max_capacity(new_soft_max);
 826       return true;
 827     }
 828   }
 829   return false;
 830 }
 831 
 832 void ShenandoahHeap::notify_heap_changed() {
 833   // Update monitoring counters when we took a new region. This amortizes the
 834   // update costs on slow path.
 835   monitoring_support()->notify_heap_changed();
 836 
 837   // This is called from allocation path, and thus should be fast.
 838   _heap_changed.try_set();
 839 }
 840 
 841 void ShenandoahHeap::set_forced_counters_update(bool value) {
 842   monitoring_support()->set_forced_counters_update(value);
 843 }
 844 
 845 void ShenandoahHeap::handle_force_counters_update() {
 846   monitoring_support()->handle_force_counters_update();
 847 }
 848 
 849 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 850   // New object should fit the GCLAB size
 851   size_t min_size = MAX2(size, PLAB::min_size());
 852 
 853   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 854   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;

 855   new_size = MIN2(new_size, PLAB::max_size());
 856   new_size = MAX2(new_size, PLAB::min_size());
 857 
 858   // Record new heuristic value even if we take any shortcut. This captures
 859   // the case when moderately-sized objects always take a shortcut. At some point,
 860   // heuristics should catch up with them.

 861   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 862 
 863   if (new_size < size) {
 864     // New size still does not fit the object. Fall back to shared allocation.
 865     // This avoids retiring perfectly good GCLABs, when we encounter a large object.

 866     return nullptr;
 867   }
 868 
 869   // Retire current GCLAB, and allocate a new one.
 870   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 871   gclab->retire();
 872 
 873   size_t actual_size = 0;
 874   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 875   if (gclab_buf == nullptr) {
 876     return nullptr;
 877   }
 878 
 879   assert (size <= actual_size, "allocation should fit");
 880 
 881   // ...and clear or zap just allocated TLAB, if needed.
 882   if (ZeroTLAB) {
 883     Copy::zero_to_words(gclab_buf, actual_size);
 884   } else if (ZapTLAB) {
 885     // Skip mangling the space corresponding to the object header to
 886     // ensure that the returned space is not considered parsable by
 887     // any concurrent GC thread.
 888     size_t hdr_size = oopDesc::header_size();
 889     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 890   }
 891   gclab->set_buf(gclab_buf, actual_size);
 892   return gclab->allocate(size);
 893 }
 894 

 895 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 896                                             size_t requested_size,
 897                                             size_t* actual_size) {
 898   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 899   HeapWord* res = allocate_memory(req);
 900   if (res != nullptr) {
 901     *actual_size = req.actual_size();
 902   } else {
 903     *actual_size = 0;
 904   }
 905   return res;
 906 }
 907 
 908 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 909                                              size_t word_size,
 910                                              size_t* actual_size) {
 911   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 912   HeapWord* res = allocate_memory(req);
 913   if (res != nullptr) {
 914     *actual_size = req.actual_size();

 958       control_thread()->handle_alloc_failure(req, true);
 959       result = allocate_memory_under_lock(req, in_new_region);
 960     }
 961 
 962     if (log_is_enabled(Debug, gc, alloc)) {
 963       ResourceMark rm;
 964       log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT ", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
 965                            Thread::current()->name(), p2i(result), req.type_string(), req.size(), original_count, get_gc_no_progress_count());
 966     }
 967   } else {
 968     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 969     result = allocate_memory_under_lock(req, in_new_region);
 970     // Do not call handle_alloc_failure() here, because we cannot block.
 971     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 972   }
 973 
 974   if (in_new_region) {
 975     notify_heap_changed();
 976   }
 977 








 978   if (result != nullptr) {
 979     size_t requested = req.size();
 980     size_t actual = req.actual_size();
 981 
 982     assert (req.is_lab_alloc() || (requested == actual),
 983             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 984             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 985 
 986     if (req.is_mutator_alloc()) {
 987       notify_mutator_alloc_words(actual, false);
 988 
 989       // If we requested more than we were granted, give the rest back to pacer.
 990       // This only matters if we are in the same pacing epoch: do not try to unpace
 991       // over the budget for the other phase.
 992       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 993         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 994       }
 995     } else {
 996       increase_used(actual*HeapWordSize);
 997     }
 998   }
 999 
1000   return result;
1001 }
1002 
1003 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1004   // If we are dealing with mutator allocation, then we may need to block for safepoint.
1005   // We cannot block for safepoint for GC allocations, because there is a high chance
1006   // we are already running at safepoint or from stack watermark machinery, and we cannot
1007   // block again.
1008   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1009   return _free_set->allocate(req, in_new_region);




































1010 }
1011 
1012 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1013                                         bool*  gc_overhead_limit_was_exceeded) {
1014   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1015   return allocate_memory(req);
1016 }
1017 
1018 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1019                                                              size_t size,
1020                                                              Metaspace::MetadataType mdtype) {
1021   MetaWord* result;
1022 
1023   // Inform metaspace OOM to GC heuristics if class unloading is possible.
1024   if (heuristics()->can_unload_classes()) {
1025     ShenandoahHeuristics* h = heuristics();
1026     h->record_metaspace_oom();
1027   }
1028 
1029   // Expand and retry allocation
1030   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1031   if (result != nullptr) {
1032     return result;
1033   }
1034 
1035   // Start full GC
1036   collect(GCCause::_metadata_GC_clear_soft_refs);
1037 
1038   // Retry allocation
1039   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1040   if (result != nullptr) {
1041     return result;
1042   }
1043 
1044   // Expand and retry allocation
1045   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

1098 private:
1099   void do_work() {
1100     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1101     ShenandoahHeapRegion* r;
1102     while ((r =_cs->claim_next()) != nullptr) {
1103       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
1104       _sh->marked_object_iterate(r, &cl);
1105 
1106       if (ShenandoahPacing) {
1107         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1108       }
1109 
1110       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1111         break;
1112       }
1113     }
1114   }
1115 };
1116 
1117 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1118   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1119   workers()->run_task(&task);






1120 }
1121 
1122 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1123   if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
1124     // This thread went through the OOM during evac protocol and it is safe to return
1125     // the forward pointer. It must not attempt to evacuate any more.

1126     return ShenandoahBarrierSet::resolve_forwarded(p);
1127   }
1128 
1129   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1130 
1131   size_t size = p->size();

1132 
1133   assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");


1134 
1135   bool alloc_from_gclab = true;




1136   HeapWord* copy = nullptr;

1137 
1138 #ifdef ASSERT
1139   if (ShenandoahOOMDuringEvacALot &&
1140       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1141     copy = nullptr;
1142   } else {
1143 #endif
1144     if (UseTLAB) {
1145       copy = allocate_from_gclab(thread, size);








1146     }

1147     if (copy == nullptr) {
1148       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);

1149       copy = allocate_memory(req);
1150       alloc_from_gclab = false;
1151     }
1152 #ifdef ASSERT
1153   }
1154 #endif
1155 
1156   if (copy == nullptr) {
1157     control_thread()->handle_alloc_failure_evac(size);
1158 
1159     _oom_evac_handler.handle_out_of_memory_during_evacuation();
1160 
1161     return ShenandoahBarrierSet::resolve_forwarded(p);
1162   }
1163 
1164   // Copy the object:

1165   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1166 
1167   // Try to install the new forwarding pointer.
1168   oop copy_val = cast_to_oop(copy);


1169   ContinuationGCSupport::relativize_stack_chunk(copy_val);
1170 
1171   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1172   if (result == copy_val) {
1173     // Successfully evacuated. Our copy is now the public one!

1174     shenandoah_assert_correct(nullptr, copy_val);
1175     return copy_val;
1176   }  else {
1177     // Failed to evacuate. We need to deal with the object that is left behind. Since this
1178     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1179     // But if it happens to contain references to evacuated regions, those references would
1180     // not get updated for this stale copy during this cycle, and we will crash while scanning
1181     // it the next cycle.
1182     //
1183     // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
1184     // object will overwrite this stale copy, or the filler object on LAB retirement will
1185     // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
1186     // have to explicitly overwrite the copy with the filler object. With that overwrite,
1187     // we have to keep the fwdptr initialized and pointing to our (stale) copy.
1188     if (alloc_from_gclab) {
1189       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
1190     } else {




1191       fill_with_object(copy, size);
1192       shenandoah_assert_correct(nullptr, copy_val);

1193     }
1194     shenandoah_assert_correct(nullptr, result);
1195     return result;
1196   }
1197 }
1198 
1199 void ShenandoahHeap::trash_cset_regions() {
1200   ShenandoahHeapLocker locker(lock());
1201 
1202   ShenandoahCollectionSet* set = collection_set();
1203   ShenandoahHeapRegion* r;
1204   set->clear_current_index();
1205   while ((r = set->next()) != nullptr) {
1206     r->make_trash();
1207   }
1208   collection_set()->clear();
1209 }
1210 
1211 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1212   st->print_cr("Heap Regions:");
1213   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1214   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1215   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1216   st->print_cr("UWM=update watermark, U=used");
1217   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1218   st->print_cr("S=shared allocs, L=live data");
1219   st->print_cr("CP=critical pins");
1220 
1221   for (size_t i = 0; i < num_regions(); i++) {
1222     get_region(i)->print_on(st);
1223   }
1224 }
1225 
1226 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1227   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1228 
1229   oop humongous_obj = cast_to_oop(start->bottom());
1230   size_t size = humongous_obj->size();
1231   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1232   size_t index = start->index() + required_regions - 1;
1233 
1234   assert(!start->has_live(), "liveness must be zero");
1235 
1236   for(size_t i = 0; i < required_regions; i++) {
1237     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1238     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1239     ShenandoahHeapRegion* region = get_region(index --);
1240 
1241     assert(region->is_humongous(), "expect correct humongous start or continuation");
1242     assert(!region->is_cset(), "Humongous region should not be in collection set");
1243 
1244     region->make_trash_immediate();
1245   }

1246 }
1247 
1248 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1249 public:
1250   ShenandoahCheckCleanGCLABClosure() {}
1251   void do_thread(Thread* thread) {
1252     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1253     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1254     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");






1255   }
1256 };
1257 
1258 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1259 private:
1260   bool const _resize;
1261 public:
1262   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1263   void do_thread(Thread* thread) {
1264     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1265     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1266     gclab->retire();
1267     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1268       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1269     }













1270   }
1271 };
1272 
1273 void ShenandoahHeap::labs_make_parsable() {
1274   assert(UseTLAB, "Only call with UseTLAB");
1275 
1276   ShenandoahRetireGCLABClosure cl(false);
1277 
1278   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1279     ThreadLocalAllocBuffer& tlab = t->tlab();
1280     tlab.make_parsable();
1281     cl.do_thread(t);
1282   }
1283 
1284   workers()->threads_do(&cl);
1285 }
1286 
1287 void ShenandoahHeap::tlabs_retire(bool resize) {
1288   assert(UseTLAB, "Only call with UseTLAB");
1289   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1378 
1379   workers()->threads_do(tcl);
1380   if (_safepoint_workers != nullptr) {
1381     _safepoint_workers->threads_do(tcl);
1382   }
1383 }
1384 
1385 void ShenandoahHeap::print_tracing_info() const {
1386   LogTarget(Info, gc, stats) lt;
1387   if (lt.is_enabled()) {
1388     ResourceMark rm;
1389     LogStream ls(lt);
1390 
1391     phase_timings()->print_global_on(&ls);
1392 
1393     ls.cr();
1394     ls.cr();
1395 
1396     shenandoah_policy()->print_gc_stats(&ls);
1397 




1398     ls.cr();
1399     ls.cr();
1400   }
1401 }
1402 



















1403 void ShenandoahHeap::verify(VerifyOption vo) {
1404   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1405     if (ShenandoahVerify) {
1406       verifier()->verify_generic(vo);
1407     } else {
1408       // TODO: Consider allocating verification bitmaps on demand,
1409       // and turn this on unconditionally.
1410     }
1411   }
1412 }
1413 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1414   return _free_set->capacity();
1415 }
1416 
1417 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1418 private:
1419   MarkBitMap* _bitmap;
1420   ShenandoahScanObjectStack* _oop_stack;
1421   ShenandoahHeap* const _heap;
1422   ShenandoahMarkingContext* const _marking_context;

1719   } else {
1720     heap_region_iterate(blk);
1721   }
1722 }
1723 
1724 class ShenandoahRendezvousClosure : public HandshakeClosure {
1725 public:
1726   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1727   inline void do_thread(Thread* thread) {}
1728 };
1729 
1730 void ShenandoahHeap::rendezvous_threads() {
1731   ShenandoahRendezvousClosure cl;
1732   Handshake::execute(&cl);
1733 }
1734 
1735 void ShenandoahHeap::recycle_trash() {
1736   free_set()->recycle_trash();
1737 }
1738 
1739 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1740 private:
1741   ShenandoahMarkingContext* const _ctx;
1742 public:
1743   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1744 
1745   void heap_region_do(ShenandoahHeapRegion* r) {
1746     if (r->is_active()) {
1747       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1748       // anyway to capture any updates that happened since now.
1749       r->clear_live_data();
1750       _ctx->capture_top_at_mark_start(r);
1751     }
1752   }
1753 
1754   bool is_thread_safe() { return true; }
1755 };
1756 
1757 void ShenandoahHeap::prepare_gc() {
1758   reset_mark_bitmap();
1759 
1760   ShenandoahResetUpdateRegionStateClosure cl;
1761   parallel_heap_region_iterate(&cl);
1762 }
1763 
1764 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1765 private:
1766   ShenandoahMarkingContext* const _ctx;
1767   ShenandoahHeapLock* const _lock;
1768 
1769 public:
1770   ShenandoahFinalMarkUpdateRegionStateClosure() :
1771     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1772 
1773   void heap_region_do(ShenandoahHeapRegion* r) {
1774     if (r->is_active()) {
1775       // All allocations past TAMS are implicitly live, adjust the region data.
1776       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1777       HeapWord *tams = _ctx->top_at_mark_start(r);
1778       HeapWord *top = r->top();
1779       if (top > tams) {
1780         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1781       }
1782 
1783       // We are about to select the collection set, make sure it knows about
1784       // current pinning status. Also, this allows trashing more regions that
1785       // now have their pinning status dropped.
1786       if (r->is_pinned()) {
1787         if (r->pin_count() == 0) {
1788           ShenandoahHeapLocker locker(_lock);
1789           r->make_unpinned();
1790         }
1791       } else {
1792         if (r->pin_count() > 0) {
1793           ShenandoahHeapLocker locker(_lock);
1794           r->make_pinned();
1795         }
1796       }
1797 
1798       // Remember limit for updating refs. It's guaranteed that we get no
1799       // from-space-refs written from here on.
1800       r->set_update_watermark_at_safepoint(r->top());
1801     } else {
1802       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1803       assert(_ctx->top_at_mark_start(r) == r->top(),
1804              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1805     }
1806   }
1807 
1808   bool is_thread_safe() { return true; }
1809 };
1810 
1811 void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
1812   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1813   {
1814     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
1815                                          ShenandoahPhaseTimings::degen_gc_final_update_region_states);
1816     ShenandoahFinalMarkUpdateRegionStateClosure cl;
1817     parallel_heap_region_iterate(&cl);
1818 
1819     assert_pinned_region_status();
1820   }
1821 
1822   {
1823     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
1824                                          ShenandoahPhaseTimings::degen_gc_choose_cset);
1825     ShenandoahHeapLocker locker(lock());
1826     _collection_set->clear();
1827     heuristics()->choose_collection_set(_collection_set);
1828   }
1829 
1830   {
1831     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
1832                                          ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
1833     ShenandoahHeapLocker locker(lock());
1834     _free_set->rebuild();
1835   }
1836 }
1837 
1838 void ShenandoahHeap::do_class_unloading() {
1839   _unloader.unload();



1840 }
1841 
1842 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1843   // Weak refs processing
1844   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1845                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1846   ShenandoahTimingsTracker t(phase);
1847   ShenandoahGCWorkerPhase worker_phase(phase);
1848   ref_processor()->process_references(phase, workers(), false /* concurrent */);
1849 }
1850 
1851 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1852   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1853 
1854   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1855   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1856   // for future GCLABs here.
1857   if (UseTLAB) {
1858     ShenandoahGCPhase phase(concurrent ?
1859                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1860                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1861     gclabs_retire(ResizeTLAB);
1862   }
1863 
1864   _update_refs_iterator.reset();
1865 }
1866 
1867 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1868   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1869   if (_gc_state_changed) {
1870     _gc_state_changed = false;
1871     char state = gc_state();
1872     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1873       ShenandoahThreadLocalData::set_gc_state(t, state);
1874     }
1875   }
1876 }
1877 
1878 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1879   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1880   _gc_state.set_cond(mask, value);
1881   _gc_state_changed = true;
1882 }
1883 
1884 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1885   assert(!has_forwarded_objects(), "Not expected before/after mark phase");
1886   set_gc_state(MARKING, in_progress);
1887   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
















































1888 }
1889 
1890 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1891   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1892   set_gc_state(EVACUATION, in_progress);
1893 }
1894 
1895 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
1896   if (in_progress) {
1897     _concurrent_strong_root_in_progress.set();
1898   } else {
1899     _concurrent_strong_root_in_progress.unset();
1900   }
1901 }
1902 
1903 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
1904   set_gc_state(WEAK_ROOTS, cond);
1905 }
1906 
1907 GCTracer* ShenandoahHeap::tracer() {
1908   return shenandoah_policy()->tracer();
1909 }
1910 
1911 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1912   return _free_set->used();
1913 }
1914 
1915 bool ShenandoahHeap::try_cancel_gc() {
1916   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1917   return prev == CANCELLABLE;
1918 }
1919 











1920 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1921   if (try_cancel_gc()) {
1922     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1923     log_info(gc)("%s", msg.buffer());
1924     Events::log(Thread::current(), "%s", msg.buffer());

1925   }
1926 }
1927 
1928 uint ShenandoahHeap::max_workers() {
1929   return _max_workers;
1930 }
1931 
1932 void ShenandoahHeap::stop() {
1933   // The shutdown sequence should be able to terminate when GC is running.
1934 
1935   // Step 0. Notify policy to disable event recording and prevent visiting gc threads during shutdown
1936   _shenandoah_policy->record_shutdown();
1937 
1938   // Step 1. Notify control thread that we are in shutdown.
1939   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1940   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1941   control_thread()->prepare_for_graceful_shutdown();
1942 
1943   // Step 2. Notify GC workers that we are cancelling GC.
1944   cancel_gc(GCCause::_shenandoah_stop_vm);

2028 }
2029 
2030 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2031   set_gc_state(HAS_FORWARDED, cond);
2032 }
2033 
2034 void ShenandoahHeap::set_unload_classes(bool uc) {
2035   _unload_classes.set_cond(uc);
2036 }
2037 
2038 bool ShenandoahHeap::unload_classes() const {
2039   return _unload_classes.is_set();
2040 }
2041 
2042 address ShenandoahHeap::in_cset_fast_test_addr() {
2043   ShenandoahHeap* heap = ShenandoahHeap::heap();
2044   assert(heap->collection_set() != nullptr, "Sanity");
2045   return (address) heap->collection_set()->biased_map_address();
2046 }
2047 
2048 size_t ShenandoahHeap::bytes_allocated_since_gc_start() const {
2049   return Atomic::load(&_bytes_allocated_since_gc_start);
2050 }
2051 
2052 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2053   Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);





2054 }
2055 
2056 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2057   _degenerated_gc_in_progress.set_cond(in_progress);
2058 }
2059 
2060 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2061   _full_gc_in_progress.set_cond(in_progress);
2062 }
2063 
2064 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2065   assert (is_full_gc_in_progress(), "should be");
2066   _full_gc_move_in_progress.set_cond(in_progress);
2067 }
2068 
2069 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2070   set_gc_state(UPDATEREFS, in_progress);
2071 }
2072 
2073 void ShenandoahHeap::register_nmethod(nmethod* nm) {

2097     if (r->is_active()) {
2098       if (r->is_pinned()) {
2099         if (r->pin_count() == 0) {
2100           r->make_unpinned();
2101         }
2102       } else {
2103         if (r->pin_count() > 0) {
2104           r->make_pinned();
2105         }
2106       }
2107     }
2108   }
2109 
2110   assert_pinned_region_status();
2111 }
2112 
2113 #ifdef ASSERT
2114 void ShenandoahHeap::assert_pinned_region_status() {
2115   for (size_t i = 0; i < num_regions(); i++) {
2116     ShenandoahHeapRegion* r = get_region(i);
2117     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2118            "Region " SIZE_FORMAT " pinning status is inconsistent", i);


2119   }
2120 }
2121 #endif
2122 
2123 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2124   return _gc_timer;
2125 }
2126 
2127 void ShenandoahHeap::prepare_concurrent_roots() {
2128   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2129   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2130   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2131   set_concurrent_weak_root_in_progress(true);
2132   if (unload_classes()) {
2133     _unloader.prepare();
2134   }
2135 }
2136 
2137 void ShenandoahHeap::finish_concurrent_roots() {
2138   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");

2153   } else {
2154     // Use ConcGCThreads outside safepoints
2155     assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2156            ConcGCThreads, nworkers);
2157   }
2158 }
2159 #endif
2160 
2161 ShenandoahVerifier* ShenandoahHeap::verifier() {
2162   guarantee(ShenandoahVerify, "Should be enabled");
2163   assert (_verifier != nullptr, "sanity");
2164   return _verifier;
2165 }
2166 
2167 template<bool CONCURRENT>
2168 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2169 private:
2170   ShenandoahHeap* _heap;
2171   ShenandoahRegionIterator* _regions;
2172 public:
2173   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2174     WorkerTask("Shenandoah Update References"),
2175     _heap(ShenandoahHeap::heap()),
2176     _regions(regions) {
2177   }
2178 
2179   void work(uint worker_id) {
2180     if (CONCURRENT) {
2181       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2182       ShenandoahSuspendibleThreadSetJoiner stsj;
2183       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
2184     } else {
2185       ShenandoahParallelWorkerSession worker_session(worker_id);
2186       do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
2187     }
2188   }
2189 
2190 private:
2191   template<class T>
2192   void do_work(uint worker_id) {
2193     T cl;
2194     if (CONCURRENT && (worker_id == 0)) {
2195       // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
2196       // results of evacuation.  These reserves are no longer necessary because evacuation has completed.
2197       size_t cset_regions = _heap->collection_set()->count();
2198       // We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled because
2199       // we need the reclaimed collection set regions to replenish the collector reserves
2200       _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
2201     }
2202     // If !CONCURRENT, there's no value in expanding Mutator free set
2203 
2204     ShenandoahHeapRegion* r = _regions->next();
2205     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2206     while (r != nullptr) {
2207       HeapWord* update_watermark = r->get_update_watermark();
2208       assert (update_watermark >= r->bottom(), "sanity");
2209       if (r->is_active() && !r->is_cset()) {
2210         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2211       }
2212       if (ShenandoahPacing) {
2213         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2214       }
2215       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2216         return;
2217       }
2218       r = _regions->next();
2219     }
2220   }
2221 };
2222 
2223 void ShenandoahHeap::update_heap_references(bool concurrent) {
2224   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2225 
2226   if (concurrent) {
2227     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2228     workers()->run_task(&task);
2229   } else {
2230     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2231     workers()->run_task(&task);
2232   }
2233 }
2234 

2235 
2236 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2237 private:
2238   ShenandoahHeapLock* const _lock;
2239 
2240 public:
2241   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2242 
2243   void heap_region_do(ShenandoahHeapRegion* r) {
2244     // Drop unnecessary "pinned" state from regions that does not have CP marks
2245     // anymore, as this would allow trashing them.
2246 
2247     if (r->is_active()) {
2248       if (r->is_pinned()) {
2249         if (r->pin_count() == 0) {
2250           ShenandoahHeapLocker locker(_lock);
2251           r->make_unpinned();
2252         }
2253       } else {
2254         if (r->pin_count() > 0) {
2255           ShenandoahHeapLocker locker(_lock);
2256           r->make_pinned();
2257         }
2258       }
2259     }
2260   }
2261 
2262   bool is_thread_safe() { return true; }
2263 };
2264 
2265 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2266   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2267   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2268 
2269   {
2270     ShenandoahGCPhase phase(concurrent ?
2271                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2272                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2273     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2274     parallel_heap_region_iterate(&cl);
2275 
2276     assert_pinned_region_status();
2277   }
2278 
2279   {
2280     ShenandoahGCPhase phase(concurrent ?
2281                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2282                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2283     trash_cset_regions();
2284   }
2285 }
2286 





2287 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2288   {
2289     ShenandoahGCPhase phase(concurrent ?
2290                             ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2291                             ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2292     ShenandoahHeapLocker locker(lock());
2293     _free_set->rebuild();




































2294   }
2295 }
2296 
2297 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2298   print_on(st);
2299   st->cr();
2300   print_heap_regions_on(st);
2301 }
2302 
2303 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2304   size_t slice = r->index() / _bitmap_regions_per_slice;
2305 
2306   size_t regions_from = _bitmap_regions_per_slice * slice;
2307   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2308   for (size_t g = regions_from; g < regions_to; g++) {
2309     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2310     if (skip_self && g == r->index()) continue;
2311     if (get_region(g)->is_committed()) {
2312       return true;
2313     }

2396 void ShenandoahHeap::initialize_serviceability() {
2397   _memory_pool = new ShenandoahMemoryPool(this);
2398   _cycle_memory_manager.add_pool(_memory_pool);
2399   _stw_memory_manager.add_pool(_memory_pool);
2400 }
2401 
2402 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2403   GrowableArray<GCMemoryManager*> memory_managers(2);
2404   memory_managers.append(&_cycle_memory_manager);
2405   memory_managers.append(&_stw_memory_manager);
2406   return memory_managers;
2407 }
2408 
2409 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2410   GrowableArray<MemoryPool*> memory_pools(1);
2411   memory_pools.append(_memory_pool);
2412   return memory_pools;
2413 }
2414 
2415 MemoryUsage ShenandoahHeap::memory_usage() {
2416   return _memory_pool->get_memory_usage();
2417 }
2418 
2419 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2420   _heap(ShenandoahHeap::heap()),
2421   _index(0) {}
2422 
2423 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2424   _heap(heap),
2425   _index(0) {}
2426 
2427 void ShenandoahRegionIterator::reset() {
2428   _index = 0;
2429 }
2430 
2431 bool ShenandoahRegionIterator::has_next() const {
2432   return _index < _heap->num_regions();
2433 }
2434 
2435 char ShenandoahHeap::gc_state() const {
2436   return _gc_state.raw_value();

2461   }
2462 }
2463 
2464 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2465   if (is_idle()) return false;
2466 
2467   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2468   // marking phase.
2469   if (is_concurrent_mark_in_progress() &&
2470      !marking_context()->allocated_after_mark_start(obj)) {
2471     return true;
2472   }
2473 
2474   // Can not guarantee obj is deeply good.
2475   if (has_forwarded_objects()) {
2476     return true;
2477   }
2478 
2479   return false;
2480 }























   1 /*
   2  * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
   4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "memory/allocation.hpp"
  29 #include "memory/universe.hpp"
  30 
  31 #include "gc/shared/classUnloadingContext.hpp"
  32 #include "gc/shared/gcArguments.hpp"
  33 #include "gc/shared/gcTimer.hpp"
  34 #include "gc/shared/gcTraceTime.inline.hpp"
  35 #include "gc/shared/locationPrinter.inline.hpp"
  36 #include "gc/shared/memAllocator.hpp"
  37 #include "gc/shared/plab.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 
  40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
  41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
  42 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  43 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  44 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  45 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  46 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  47 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  48 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  49 #include "gc/shenandoah/shenandoahControlThread.hpp"
  50 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  51 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
  52 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  53 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
  54 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  55 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  56 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  57 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  58 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  59 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  60 #include "gc/shenandoah/shenandoahMemoryPool.hpp"

  61 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  62 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  63 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  64 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  65 #include "gc/shenandoah/shenandoahPadding.hpp"
  66 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  67 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  68 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  69 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  70 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  71 #include "gc/shenandoah/shenandoahUtils.hpp"
  72 #include "gc/shenandoah/shenandoahVerifier.hpp"
  73 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  74 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  75 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  76 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  77 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  78 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
  79 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  80 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  81 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  82 #include "utilities/globalDefinitions.hpp"
  83 
  84 #if INCLUDE_JFR
  85 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  86 #endif
  87 
  88 #include "classfile/systemDictionary.hpp"
  89 #include "code/codeCache.hpp"
  90 #include "memory/classLoaderMetaspace.hpp"
  91 #include "memory/metaspaceUtils.hpp"
  92 #include "nmt/mallocTracker.hpp"
  93 #include "nmt/memTracker.hpp"
  94 #include "oops/compressedOops.inline.hpp"
  95 #include "prims/jvmtiTagMap.hpp"
  96 #include "runtime/atomic.hpp"
  97 #include "runtime/globals.hpp"
  98 #include "runtime/interfaceSupport.inline.hpp"
  99 #include "runtime/java.hpp"
 100 #include "runtime/orderAccess.hpp"
 101 #include "runtime/safepointMechanism.hpp"
 102 #include "runtime/stackWatermarkSet.hpp"
 103 #include "runtime/vmThread.hpp"

 156 jint ShenandoahHeap::initialize() {
 157   //
 158   // Figure out heap sizing
 159   //
 160 
 161   size_t init_byte_size = InitialHeapSize;
 162   size_t min_byte_size  = MinHeapSize;
 163   size_t max_byte_size  = MaxHeapSize;
 164   size_t heap_alignment = HeapAlignment;
 165 
 166   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 167 
 168   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 169   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 170 
 171   _num_regions = ShenandoahHeapRegion::region_count();
 172   assert(_num_regions == (max_byte_size / reg_size_bytes),
 173          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 174          _num_regions, max_byte_size, reg_size_bytes);
 175 



 176   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 177   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 178   assert(num_committed_regions <= _num_regions, "sanity");
 179   _initial_size = num_committed_regions * reg_size_bytes;
 180 
 181   size_t num_min_regions = min_byte_size / reg_size_bytes;
 182   num_min_regions = MIN2(num_min_regions, _num_regions);
 183   assert(num_min_regions <= _num_regions, "sanity");
 184   _minimum_size = num_min_regions * reg_size_bytes;
 185 
 186   // Default to max heap size.
 187   _soft_max_size = _num_regions * reg_size_bytes;
 188 
 189   _committed = _initial_size;
 190 
 191   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 192   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 193   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 194 
 195   //

 209                                           heap_rs.size(), heap_rs.page_size());
 210 
 211 #if SHENANDOAH_OPTIMIZED_MARKTASK
 212   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 213   // Fail if we ever attempt to address more than we can.
 214   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 215     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 216                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 217                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 218                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 219     vm_exit_during_initialization("Fatal Error", buf);
 220   }
 221 #endif
 222 
 223   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 224   if (!_heap_region_special) {
 225     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 226                               "Cannot commit heap memory");
 227   }
 228 
 229   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
 230 
 231   // Now we know the number of regions and heap sizes, initialize the heuristics.
 232   initialize_heuristics();
 233 
 234   assert(_heap_region.byte_size() == heap_rs.size(), "Need to know reserved size for card table");
 235 
 236   //
 237   // Worker threads must be initialized after the barrier is configured
 238   //
 239   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 240   if (_workers == nullptr) {
 241     vm_exit_during_initialization("Failed necessary allocation.");
 242   } else {
 243     _workers->initialize_workers();
 244   }
 245 
 246   if (ParallelGCThreads > 1) {
 247     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread", ParallelGCThreads);
 248     _safepoint_workers->initialize_workers();
 249   }
 250 
 251   //
 252   // Reserve and commit memory for bitmap(s)
 253   //
 254 
 255   size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 256   _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
 257 
 258   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 259 
 260   guarantee(bitmap_bytes_per_region != 0,
 261             "Bitmap bytes per region should not be zero");
 262   guarantee(is_power_of_2(bitmap_bytes_per_region),
 263             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 264 
 265   if (bitmap_page_size > bitmap_bytes_per_region) {
 266     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 267     _bitmap_bytes_per_slice = bitmap_page_size;
 268   } else {
 269     _bitmap_regions_per_slice = 1;
 270     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 271   }
 272 
 273   guarantee(_bitmap_regions_per_slice >= 1,
 274             "Should have at least one region per slice: " SIZE_FORMAT,
 275             _bitmap_regions_per_slice);
 276 
 277   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 278             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 279             _bitmap_bytes_per_slice, bitmap_page_size);
 280 
 281   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 282   os::trace_page_sizes_for_requested_size("Mark Bitmap",
 283                                           bitmap_size_orig, bitmap_page_size,
 284                                           bitmap.base(),
 285                                           bitmap.size(), bitmap.page_size());
 286   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 287   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 288   _bitmap_region_special = bitmap.special();
 289 
 290   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 291     align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 292   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 293   if (!_bitmap_region_special) {
 294     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 295                               "Cannot commit bitmap memory");
 296   }
 297 
 298   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 299 
 300   if (ShenandoahVerify) {
 301     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 302     os::trace_page_sizes_for_requested_size("Verify Bitmap",
 303                                             bitmap_size_orig, bitmap_page_size,
 304                                             verify_bitmap.base(),
 305                                             verify_bitmap.size(), verify_bitmap.page_size());
 306     if (!verify_bitmap.special()) {
 307       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 308                                 "Cannot commit verification bitmap memory");
 309     }
 310     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 311     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 312     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 313     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 314   }
 315 
 316   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 317   size_t aux_bitmap_page_size = bitmap_page_size;
 318 #ifdef LINUX

 369       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 370       cset_rs = ReservedSpace(cset_size, cset_align, cset_page_size, req_addr);
 371       if (cset_rs.is_reserved()) {
 372         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 373         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 374         break;
 375       }
 376     }
 377 
 378     if (_collection_set == nullptr) {
 379       cset_rs = ReservedSpace(cset_size, cset_align, os::vm_page_size());
 380       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 381     }
 382     os::trace_page_sizes_for_requested_size("Collection Set",
 383                                             cset_size, cset_page_size,
 384                                             cset_rs.base(),
 385                                             cset_rs.size(), cset_rs.page_size());
 386   }
 387 
 388   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 389   _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC);
 390   _free_set = new ShenandoahFreeSet(this, _num_regions);
 391 
 392   {
 393     ShenandoahHeapLocker locker(lock());
 394 
 395 
 396     for (size_t i = 0; i < _num_regions; i++) {
 397       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 398       bool is_committed = i < num_committed_regions;
 399       void* loc = region_storage.base() + i * region_align;
 400 
 401       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 402       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 403 
 404       _marking_context->initialize_top_at_mark_start(r);
 405       _regions[i] = r;
 406       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 407 
 408       _affiliations[i] = ShenandoahAffiliation::FREE;
 409     }
 410 
 411     // Initialize to complete
 412     _marking_context->mark_complete();
 413     size_t young_cset_regions, old_cset_regions;
 414 
 415     // We are initializing free set.  We ignore cset region tallies.
 416     size_t first_old, last_old, num_old;
 417     _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
 418     _free_set->rebuild(young_cset_regions, old_cset_regions);
 419   }
 420 
 421   if (AlwaysPreTouch) {
 422     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 423     // before initialize() below zeroes it with initializing thread. For any given region,
 424     // we touch the region and the corresponding bitmaps from the same thread.
 425     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 426 
 427     _pretouch_heap_page_size = heap_page_size;
 428     _pretouch_bitmap_page_size = bitmap_page_size;
 429 
 430 #ifdef LINUX
 431     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 432     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 433     // them into huge one. Therefore, we need to pretouch with smaller pages.
 434     if (UseTransparentHugePages) {
 435       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 436       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 437     }
 438 #endif

 457     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 458   }
 459 
 460   // There should probably be Shenandoah-specific options for these,
 461   // just as there are G1-specific options.
 462   {
 463     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 464     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 465     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 466   }
 467 
 468   _monitoring_support = new ShenandoahMonitoringSupport(this);
 469   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 470   ShenandoahCodeRoots::initialize();
 471 
 472   if (ShenandoahPacing) {
 473     _pacer = new ShenandoahPacer(this);
 474     _pacer->setup_for_idle();
 475   }
 476 
 477   initialize_controller();
 478 
 479   print_init_logger();
 480 
 481   return JNI_OK;
 482 }
 483 
 484 void ShenandoahHeap::initialize_controller() {
 485   _control_thread = new ShenandoahControlThread();
 486 }
 487 
 488 void ShenandoahHeap::print_init_logger() const {
 489   ShenandoahInitLogger::print();
 490 }
 491 
 492 void ShenandoahHeap::initialize_mode() {
 493   if (ShenandoahGCMode != nullptr) {
 494     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 495       _gc_mode = new ShenandoahSATBMode();
 496     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 497       _gc_mode = new ShenandoahIUMode();
 498     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 499       _gc_mode = new ShenandoahPassiveMode();
 500     } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
 501       _gc_mode = new ShenandoahGenerationalMode();
 502     } else {
 503       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 504     }
 505   } else {
 506     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 507   }
 508   _gc_mode->initialize_flags();
 509   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 510     vm_exit_during_initialization(
 511             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 512                     _gc_mode->name()));
 513   }
 514   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 515     vm_exit_during_initialization(
 516             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 517                     _gc_mode->name()));
 518   }
 519 }
 520 
 521 void ShenandoahHeap::initialize_heuristics() {
 522   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
 523   _global_generation->initialize_heuristics(mode());
 524   _evac_tracker = new ShenandoahEvacuationTracker(mode()->is_generational());










 525 }
 526 
 527 #ifdef _MSC_VER
 528 #pragma warning( push )
 529 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 530 #endif
 531 
 532 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 533   CollectedHeap(),
 534   _gc_generation(nullptr),
 535   _initial_size(0),

 536   _committed(0),
 537   _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),

 538   _workers(nullptr),
 539   _safepoint_workers(nullptr),
 540   _heap_region_special(false),
 541   _num_regions(0),
 542   _regions(nullptr),
 543   _affiliations(nullptr),
 544   _gc_state_changed(false),
 545   _gc_no_progress_count(0),
 546   _cancel_requested_time(0),
 547   _update_refs_iterator(this),
 548   _global_generation(nullptr),
 549   _control_thread(nullptr),
 550   _young_generation(nullptr),
 551   _old_generation(nullptr),
 552   _shenandoah_policy(policy),
 553   _gc_mode(nullptr),

 554   _free_set(nullptr),
 555   _pacer(nullptr),
 556   _verifier(nullptr),
 557   _phase_timings(nullptr),
 558   _evac_tracker(nullptr),
 559   _mmu_tracker(),
 560   _monitoring_support(nullptr),
 561   _memory_pool(nullptr),
 562   _stw_memory_manager("Shenandoah Pauses"),
 563   _cycle_memory_manager("Shenandoah Cycles"),
 564   _gc_timer(new ConcurrentGCTimer()),
 565   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),

 566   _marking_context(nullptr),
 567   _bitmap_size(0),
 568   _bitmap_regions_per_slice(0),
 569   _bitmap_bytes_per_slice(0),
 570   _bitmap_region_special(false),
 571   _aux_bitmap_region_special(false),
 572   _liveness_cache(nullptr),
 573   _collection_set(nullptr)
 574 {
 575   // Initialize GC mode early, many subsequent initialization procedures depend on it
 576   initialize_mode();















 577 }
 578 
 579 #ifdef _MSC_VER
 580 #pragma warning( pop )
 581 #endif
 582 





























 583 void ShenandoahHeap::print_on(outputStream* st) const {
 584   st->print_cr("Shenandoah Heap");
 585   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 586                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 587                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 588                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 589                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 590   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 591                num_regions(),
 592                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 593                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 594 
 595   st->print("Status: ");
 596   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 597   if (is_concurrent_old_mark_in_progress())    st->print("old marking, ");
 598   if (is_concurrent_young_mark_in_progress())  st->print("young marking, ");
 599   if (is_evacuation_in_progress())             st->print("evacuating, ");
 600   if (is_update_refs_in_progress())            st->print("updating refs, ");
 601   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 602   if (is_full_gc_in_progress())                st->print("full gc, ");
 603   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 604   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 605   if (is_concurrent_strong_root_in_progress() &&
 606       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 607 
 608   if (cancelled_gc()) {
 609     st->print("cancelled");
 610   } else {
 611     st->print("not cancelled");
 612   }
 613   st->cr();
 614 
 615   st->print_cr("Reserved region:");
 616   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 617                p2i(reserved_region().start()),
 618                p2i(reserved_region().end()));

 629   st->cr();
 630   MetaspaceUtils::print_on(st);
 631 
 632   if (Verbose) {
 633     st->cr();
 634     print_heap_regions_on(st);
 635   }
 636 }
 637 
 638 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 639 public:
 640   void do_thread(Thread* thread) {
 641     assert(thread != nullptr, "Sanity");
 642     assert(thread->is_Worker_thread(), "Only worker thread expected");
 643     ShenandoahThreadLocalData::initialize_gclab(thread);
 644   }
 645 };
 646 
 647 void ShenandoahHeap::post_initialize() {
 648   CollectedHeap::post_initialize();
 649   _mmu_tracker.initialize();
 650 
 651   MutexLocker ml(Threads_lock);
 652 
 653   ShenandoahInitWorkerGCLABClosure init_gclabs;
 654   _workers->threads_do(&init_gclabs);
 655 
 656   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 657   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 658   _workers->set_initialize_gclab();
 659   if (_safepoint_workers != nullptr) {
 660     _safepoint_workers->threads_do(&init_gclabs);
 661     _safepoint_workers->set_initialize_gclab();
 662   }
 663 


 664   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
 665 }
 666 
 667 ShenandoahHeuristics* ShenandoahHeap::heuristics() {
 668   return _global_generation->heuristics();
 669 }
 670 
 671 size_t ShenandoahHeap::used() const {
 672   return global_generation()->used();
 673 }
 674 
 675 size_t ShenandoahHeap::committed() const {
 676   return Atomic::load(&_committed);
 677 }
 678 




 679 void ShenandoahHeap::increase_committed(size_t bytes) {
 680   shenandoah_assert_heaplocked_or_safepoint();
 681   _committed += bytes;
 682 }
 683 
 684 void ShenandoahHeap::decrease_committed(size_t bytes) {
 685   shenandoah_assert_heaplocked_or_safepoint();
 686   _committed -= bytes;
 687 }
 688 
 689 // For tracking usage based on allocations, it should be the case that:
 690 // * The sum of regions::used == heap::used
 691 // * The sum of a generation's regions::used == generation::used
 692 // * The sum of a generation's humongous regions::free == generation::humongous_waste
 693 // These invariants are checked by the verifier on GC safepoints.
 694 //
 695 // Additional notes:
 696 // * When a mutator's allocation request causes a region to be retired, the
 697 //   free memory left in that region is considered waste. It does not contribute
 698 //   to the usage, but it _does_ contribute to allocation rate.
 699 // * The bottom of a PLAB must be aligned on card size. In some cases this will
 700 //   require padding in front of the PLAB (a filler object). Because this padding
 701 //   is included in the region's used memory we include the padding in the usage
 702 //   accounting as waste.
 703 // * Mutator allocations are used to compute an allocation rate. They are also
 704 //   sent to the Pacer for those purposes.
 705 // * There are three sources of waste:
 706 //  1. The padding used to align a PLAB on card size
 707 //  2. Region's free is less than minimum TLAB size and is retired
 708 //  3. The unused portion of memory in the last region of a humongous object
 709 void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) {
 710   size_t actual_bytes = req.actual_size() * HeapWordSize;
 711   size_t wasted_bytes = req.waste() * HeapWordSize;
 712   ShenandoahGeneration* generation = generation_for(req.affiliation());
 713 
 714   if (req.is_gc_alloc()) {
 715     assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste");
 716     increase_used(generation, actual_bytes + wasted_bytes);
 717   } else {
 718     assert(req.is_mutator_alloc(), "Expected mutator alloc here");
 719     // padding and actual size both count towards allocation counter
 720     generation->increase_allocated(actual_bytes + wasted_bytes);
 721 
 722     // only actual size counts toward usage for mutator allocations
 723     increase_used(generation, actual_bytes);
 724 
 725     // notify pacer of both actual size and waste
 726     notify_mutator_alloc_words(req.actual_size(), req.waste());
 727 
 728     if (wasted_bytes > 0 && req.actual_size() > ShenandoahHeapRegion::humongous_threshold_words()) {
 729       increase_humongous_waste(generation,wasted_bytes);
 730     }
 731   }
 732 }
 733 
 734 void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 735   generation->increase_humongous_waste(bytes);
 736   if (!generation->is_global()) {
 737     global_generation()->increase_humongous_waste(bytes);
 738   }
 739 }
 740 
 741 void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 742   generation->decrease_humongous_waste(bytes);
 743   if (!generation->is_global()) {
 744     global_generation()->decrease_humongous_waste(bytes);
 745   }
 746 }
 747 
 748 void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) {
 749   generation->increase_used(bytes);
 750   if (!generation->is_global()) {
 751     global_generation()->increase_used(bytes);
 752   }
 753 }
 754 
 755 void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) {
 756   generation->decrease_used(bytes);
 757   if (!generation->is_global()) {
 758     global_generation()->decrease_used(bytes);
 759   }
 760 }
 761 
 762 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) {
 763   if (ShenandoahPacing) {
 764     control_thread()->pacing_notify_alloc(words);
 765     if (waste > 0) {
 766       pacer()->claim_for_alloc(waste, true);
 767     }
 768   }
 769 }
 770 
 771 size_t ShenandoahHeap::capacity() const {
 772   return committed();
 773 }
 774 
 775 size_t ShenandoahHeap::max_capacity() const {
 776   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 777 }
 778 
 779 size_t ShenandoahHeap::soft_max_capacity() const {
 780   size_t v = Atomic::load(&_soft_max_size);
 781   assert(min_capacity() <= v && v <= max_capacity(),
 782          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 783          min_capacity(), v, max_capacity());
 784   return v;
 785 }
 786 
 787 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 788   assert(min_capacity() <= v && v <= max_capacity(),
 789          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 790          min_capacity(), v, max_capacity());
 791   Atomic::store(&_soft_max_size, v);
 792 }
 793 
 794 size_t ShenandoahHeap::min_capacity() const {
 795   return _minimum_size;
 796 }
 797 
 798 size_t ShenandoahHeap::initial_capacity() const {
 799   return _initial_size;
 800 }
 801 






 802 void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
 803   assert (ShenandoahUncommit, "should be enabled");
 804 
 805   // Determine if there is work to do. This avoids taking heap lock if there is
 806   // no work available, avoids spamming logs with superfluous logging messages,
 807   // and minimises the amount of work while locks are taken.
 808 
 809   if (committed() <= shrink_until) return;
 810 
 811   bool has_work = false;
 812   for (size_t i = 0; i < num_regions(); i++) {
 813     ShenandoahHeapRegion* r = get_region(i);
 814     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 815       has_work = true;
 816       break;
 817     }
 818   }
 819 
 820   if (has_work) {
 821     static const char* msg = "Concurrent uncommit";

 861   size_t old_soft_max = soft_max_capacity();
 862   if (new_soft_max != old_soft_max) {
 863     new_soft_max = MAX2(min_capacity(), new_soft_max);
 864     new_soft_max = MIN2(max_capacity(), new_soft_max);
 865     if (new_soft_max != old_soft_max) {
 866       log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
 867                    byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
 868                    byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
 869       );
 870       set_soft_max_capacity(new_soft_max);
 871       return true;
 872     }
 873   }
 874   return false;
 875 }
 876 
 877 void ShenandoahHeap::notify_heap_changed() {
 878   // Update monitoring counters when we took a new region. This amortizes the
 879   // update costs on slow path.
 880   monitoring_support()->notify_heap_changed();
 881   _heap_changed.set();


 882 }
 883 
 884 void ShenandoahHeap::set_forced_counters_update(bool value) {
 885   monitoring_support()->set_forced_counters_update(value);
 886 }
 887 
 888 void ShenandoahHeap::handle_force_counters_update() {
 889   monitoring_support()->handle_force_counters_update();
 890 }
 891 
 892 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 893   // New object should fit the GCLAB size
 894   size_t min_size = MAX2(size, PLAB::min_size());
 895 
 896   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 897   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 898 
 899   new_size = MIN2(new_size, PLAB::max_size());
 900   new_size = MAX2(new_size, PLAB::min_size());
 901 
 902   // Record new heuristic value even if we take any shortcut. This captures
 903   // the case when moderately-sized objects always take a shortcut. At some point,
 904   // heuristics should catch up with them.
 905   log_debug(gc, free)("Set new GCLAB size: " SIZE_FORMAT, new_size);
 906   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 907 
 908   if (new_size < size) {
 909     // New size still does not fit the object. Fall back to shared allocation.
 910     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 911     log_debug(gc, free)("New gclab size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, new_size, size);
 912     return nullptr;
 913   }
 914 
 915   // Retire current GCLAB, and allocate a new one.
 916   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 917   gclab->retire();
 918 
 919   size_t actual_size = 0;
 920   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 921   if (gclab_buf == nullptr) {
 922     return nullptr;
 923   }
 924 
 925   assert (size <= actual_size, "allocation should fit");
 926 
 927   // ...and clear or zap just allocated TLAB, if needed.
 928   if (ZeroTLAB) {
 929     Copy::zero_to_words(gclab_buf, actual_size);
 930   } else if (ZapTLAB) {
 931     // Skip mangling the space corresponding to the object header to
 932     // ensure that the returned space is not considered parsable by
 933     // any concurrent GC thread.
 934     size_t hdr_size = oopDesc::header_size();
 935     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 936   }
 937   gclab->set_buf(gclab_buf, actual_size);
 938   return gclab->allocate(size);
 939 }
 940 
 941 // Called from stubs in JIT code or interpreter
 942 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 943                                             size_t requested_size,
 944                                             size_t* actual_size) {
 945   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 946   HeapWord* res = allocate_memory(req);
 947   if (res != nullptr) {
 948     *actual_size = req.actual_size();
 949   } else {
 950     *actual_size = 0;
 951   }
 952   return res;
 953 }
 954 
 955 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 956                                              size_t word_size,
 957                                              size_t* actual_size) {
 958   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 959   HeapWord* res = allocate_memory(req);
 960   if (res != nullptr) {
 961     *actual_size = req.actual_size();

1005       control_thread()->handle_alloc_failure(req, true);
1006       result = allocate_memory_under_lock(req, in_new_region);
1007     }
1008 
1009     if (log_is_enabled(Debug, gc, alloc)) {
1010       ResourceMark rm;
1011       log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT ", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
1012                            Thread::current()->name(), p2i(result), req.type_string(), req.size(), original_count, get_gc_no_progress_count());
1013     }
1014   } else {
1015     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
1016     result = allocate_memory_under_lock(req, in_new_region);
1017     // Do not call handle_alloc_failure() here, because we cannot block.
1018     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
1019   }
1020 
1021   if (in_new_region) {
1022     notify_heap_changed();
1023   }
1024 
1025   if (result == nullptr) {
1026     req.set_actual_size(0);
1027   }
1028 
1029   // This is called regardless of the outcome of the allocation to account
1030   // for any waste created by retiring regions with this request.
1031   increase_used(req);
1032 
1033   if (result != nullptr) {
1034     size_t requested = req.size();
1035     size_t actual = req.actual_size();
1036 
1037     assert (req.is_lab_alloc() || (requested == actual),
1038             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
1039             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
1040 
1041     if (req.is_mutator_alloc()) {


1042       // If we requested more than we were granted, give the rest back to pacer.
1043       // This only matters if we are in the same pacing epoch: do not try to unpace
1044       // over the budget for the other phase.
1045       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
1046         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
1047       }


1048     }
1049   }
1050 
1051   return result;
1052 }
1053 
1054 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1055   // If we are dealing with mutator allocation, then we may need to block for safepoint.
1056   // We cannot block for safepoint for GC allocations, because there is a high chance
1057   // we are already running at safepoint or from stack watermark machinery, and we cannot
1058   // block again.
1059   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1060 
1061   // Make sure the old generation has room for either evacuations or promotions before trying to allocate.
1062   if (req.is_old() && !old_generation()->can_allocate(req)) {
1063     return nullptr;
1064   }
1065 
1066   // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
1067   // memory.
1068   HeapWord* result = _free_set->allocate(req, in_new_region);
1069 
1070   // Record the plab configuration for this result and register the object.
1071   if (result != nullptr && req.is_old()) {
1072     old_generation()->configure_plab_for_current_thread(req);
1073     if (req.type() == ShenandoahAllocRequest::_alloc_shared_gc) {
1074       // Register the newly allocated object while we're holding the global lock since there's no synchronization
1075       // built in to the implementation of register_object().  There are potential races when multiple independent
1076       // threads are allocating objects, some of which might span the same card region.  For example, consider
1077       // a card table's memory region within which three objects are being allocated by three different threads:
1078       //
1079       // objects being "concurrently" allocated:
1080       //    [-----a------][-----b-----][--------------c------------------]
1081       //            [---- card table memory range --------------]
1082       //
1083       // Before any objects are allocated, this card's memory range holds no objects.  Note that allocation of object a
1084       // wants to set the starts-object, first-start, and last-start attributes of the preceding card region.
1085       // Allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region.
1086       // Allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this
1087       // card region.
1088       //
1089       // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as
1090       // last-start representing object b while first-start represents object c.  This is why we need to require all
1091       // register_object() invocations to be "mutually exclusive" with respect to each card's memory range.
1092       old_generation()->card_scan()->register_object(result);
1093     }
1094   }
1095 
1096   return result;
1097 }
1098 
1099 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1100                                         bool*  gc_overhead_limit_was_exceeded) {
1101   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1102   return allocate_memory(req);
1103 }
1104 
1105 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1106                                                              size_t size,
1107                                                              Metaspace::MetadataType mdtype) {
1108   MetaWord* result;
1109 
1110   // Inform metaspace OOM to GC heuristics if class unloading is possible.
1111   ShenandoahHeuristics* h = global_generation()->heuristics();
1112   if (h->can_unload_classes()) {
1113     h->record_metaspace_oom();
1114   }
1115 
1116   // Expand and retry allocation
1117   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1118   if (result != nullptr) {
1119     return result;
1120   }
1121 
1122   // Start full GC
1123   collect(GCCause::_metadata_GC_clear_soft_refs);
1124 
1125   // Retry allocation
1126   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1127   if (result != nullptr) {
1128     return result;
1129   }
1130 
1131   // Expand and retry allocation
1132   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

1185 private:
1186   void do_work() {
1187     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1188     ShenandoahHeapRegion* r;
1189     while ((r =_cs->claim_next()) != nullptr) {
1190       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
1191       _sh->marked_object_iterate(r, &cl);
1192 
1193       if (ShenandoahPacing) {
1194         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1195       }
1196 
1197       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1198         break;
1199       }
1200     }
1201   }
1202 };
1203 
1204 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1205   if (mode()->is_generational()) {
1206     ShenandoahRegionIterator regions;
1207     ShenandoahGenerationalEvacuationTask task(ShenandoahGenerationalHeap::heap(), &regions, concurrent);
1208     workers()->run_task(&task);
1209   } else {
1210     ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1211     workers()->run_task(&task);
1212   }
1213 }
1214 
1215 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1216   assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1217   if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
1218     // This thread went through the OOM during evac protocol. It is safe to return
1219     // the forward pointer. It must not attempt to evacuate any other objects.
1220     return ShenandoahBarrierSet::resolve_forwarded(p);
1221   }
1222 
1223   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1224 
1225   ShenandoahHeapRegion* r = heap_region_containing(p);
1226   assert(!r->is_humongous(), "never evacuate humongous objects");
1227 
1228   ShenandoahAffiliation target_gen = r->affiliation();
1229   return try_evacuate_object(p, thread, r, target_gen);
1230 }
1231 
1232 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1233                                                ShenandoahAffiliation target_gen) {
1234   assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1235   assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1236   bool alloc_from_lab = true;
1237   HeapWord* copy = nullptr;
1238   size_t size = p->size();
1239 
1240 #ifdef ASSERT
1241   if (ShenandoahOOMDuringEvacALot &&
1242       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1243     copy = nullptr;
1244   } else {
1245 #endif
1246     if (UseTLAB) {
1247       copy = allocate_from_gclab(thread, size);
1248       if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
1249         // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve.  Try resetting
1250         // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
1251         // TODO: is this right? using PLAB::min_size() here for gc lab size?
1252         ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
1253         copy = allocate_from_gclab(thread, size);
1254         // If we still get nullptr, we'll try a shared allocation below.
1255       }
1256     }
1257 
1258     if (copy == nullptr) {
1259       // If we failed to allocate in LAB, we'll try a shared allocation.
1260       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1261       copy = allocate_memory(req);
1262       alloc_from_lab = false;
1263     }
1264 #ifdef ASSERT
1265   }
1266 #endif
1267 
1268   if (copy == nullptr) {
1269     control_thread()->handle_alloc_failure_evac(size);
1270 
1271     _oom_evac_handler.handle_out_of_memory_during_evacuation();
1272 
1273     return ShenandoahBarrierSet::resolve_forwarded(p);
1274   }
1275 
1276   // Copy the object:
1277   _evac_tracker->begin_evacuation(thread, size * HeapWordSize);
1278   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1279 

1280   oop copy_val = cast_to_oop(copy);
1281 
1282   // Try to install the new forwarding pointer.
1283   ContinuationGCSupport::relativize_stack_chunk(copy_val);
1284 
1285   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1286   if (result == copy_val) {
1287     // Successfully evacuated. Our copy is now the public one!
1288     _evac_tracker->end_evacuation(thread, size * HeapWordSize);
1289     shenandoah_assert_correct(nullptr, copy_val);
1290     return copy_val;
1291   }  else {
1292     // Failed to evacuate. We need to deal with the object that is left behind. Since this
1293     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1294     // But if it happens to contain references to evacuated regions, those references would
1295     // not get updated for this stale copy during this cycle, and we will crash while scanning
1296     // it the next cycle.
1297     if (alloc_from_lab) {
1298       // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1299       // object will overwrite this stale copy, or the filler object on LAB retirement will
1300       // do this.



1301       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
1302     } else {
1303       // For non-LAB allocations, we have no way to retract the allocation, and
1304       // have to explicitly overwrite the copy with the filler object. With that overwrite,
1305       // we have to keep the fwdptr initialized and pointing to our (stale) copy.
1306       assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
1307       fill_with_object(copy, size);
1308       shenandoah_assert_correct(nullptr, copy_val);
1309       // For non-LAB allocations, the object has already been registered
1310     }
1311     shenandoah_assert_correct(nullptr, result);
1312     return result;
1313   }
1314 }
1315 
1316 void ShenandoahHeap::trash_cset_regions() {
1317   ShenandoahHeapLocker locker(lock());
1318 
1319   ShenandoahCollectionSet* set = collection_set();
1320   ShenandoahHeapRegion* r;
1321   set->clear_current_index();
1322   while ((r = set->next()) != nullptr) {
1323     r->make_trash();
1324   }
1325   collection_set()->clear();
1326 }
1327 
1328 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1329   st->print_cr("Heap Regions:");
1330   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1331   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1332   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1333   st->print_cr("UWM=update watermark, U=used");
1334   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1335   st->print_cr("S=shared allocs, L=live data");
1336   st->print_cr("CP=critical pins");
1337 
1338   for (size_t i = 0; i < num_regions(); i++) {
1339     get_region(i)->print_on(st);
1340   }
1341 }
1342 
1343 size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1344   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1345 
1346   oop humongous_obj = cast_to_oop(start->bottom());
1347   size_t size = humongous_obj->size();
1348   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1349   size_t index = start->index() + required_regions - 1;
1350 
1351   assert(!start->has_live(), "liveness must be zero");
1352 
1353   for(size_t i = 0; i < required_regions; i++) {
1354     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1355     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1356     ShenandoahHeapRegion* region = get_region(index --);
1357 
1358     assert(region->is_humongous(), "expect correct humongous start or continuation");
1359     assert(!region->is_cset(), "Humongous region should not be in collection set");
1360 
1361     region->make_trash_immediate();
1362   }
1363   return required_regions;
1364 }
1365 
1366 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1367 public:
1368   ShenandoahCheckCleanGCLABClosure() {}
1369   void do_thread(Thread* thread) {
1370     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1371     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1372     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1373 
1374     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1375       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1376       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1377       assert(plab->words_remaining() == 0, "PLAB should not need retirement");
1378     }
1379   }
1380 };
1381 
1382 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1383 private:
1384   bool const _resize;
1385 public:
1386   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1387   void do_thread(Thread* thread) {
1388     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1389     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1390     gclab->retire();
1391     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1392       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1393     }
1394 
1395     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1396       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1397       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1398 
1399       // There are two reasons to retire all plabs between old-gen evacuation passes.
1400       //  1. We need to make the plab memory parsable by remembered-set scanning.
1401       //  2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
1402       ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
1403       if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
1404         ShenandoahThreadLocalData::set_plab_size(thread, 0);
1405       }
1406     }
1407   }
1408 };
1409 
1410 void ShenandoahHeap::labs_make_parsable() {
1411   assert(UseTLAB, "Only call with UseTLAB");
1412 
1413   ShenandoahRetireGCLABClosure cl(false);
1414 
1415   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1416     ThreadLocalAllocBuffer& tlab = t->tlab();
1417     tlab.make_parsable();
1418     cl.do_thread(t);
1419   }
1420 
1421   workers()->threads_do(&cl);
1422 }
1423 
1424 void ShenandoahHeap::tlabs_retire(bool resize) {
1425   assert(UseTLAB, "Only call with UseTLAB");
1426   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1515 
1516   workers()->threads_do(tcl);
1517   if (_safepoint_workers != nullptr) {
1518     _safepoint_workers->threads_do(tcl);
1519   }
1520 }
1521 
1522 void ShenandoahHeap::print_tracing_info() const {
1523   LogTarget(Info, gc, stats) lt;
1524   if (lt.is_enabled()) {
1525     ResourceMark rm;
1526     LogStream ls(lt);
1527 
1528     phase_timings()->print_global_on(&ls);
1529 
1530     ls.cr();
1531     ls.cr();
1532 
1533     shenandoah_policy()->print_gc_stats(&ls);
1534 
1535     ls.cr();
1536 
1537     evac_tracker()->print_global_on(&ls);
1538 
1539     ls.cr();
1540     ls.cr();
1541   }
1542 }
1543 
1544 void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) {
1545   shenandoah_policy()->record_collection_cause(cause);
1546 
1547   set_gc_cause(cause);
1548   set_gc_generation(generation);
1549 
1550   generation->heuristics()->record_cycle_start();
1551 }
1552 
1553 void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) {
1554   generation->heuristics()->record_cycle_end();
1555   if (mode()->is_generational() && generation->is_global()) {
1556     // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well
1557     young_generation()->heuristics()->record_cycle_end();
1558     old_generation()->heuristics()->record_cycle_end();
1559   }
1560   set_gc_cause(GCCause::_no_gc);
1561 }
1562 
1563 void ShenandoahHeap::verify(VerifyOption vo) {
1564   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1565     if (ShenandoahVerify) {
1566       verifier()->verify_generic(vo);
1567     } else {
1568       // TODO: Consider allocating verification bitmaps on demand,
1569       // and turn this on unconditionally.
1570     }
1571   }
1572 }
1573 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1574   return _free_set->capacity();
1575 }
1576 
1577 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1578 private:
1579   MarkBitMap* _bitmap;
1580   ShenandoahScanObjectStack* _oop_stack;
1581   ShenandoahHeap* const _heap;
1582   ShenandoahMarkingContext* const _marking_context;

1879   } else {
1880     heap_region_iterate(blk);
1881   }
1882 }
1883 
1884 class ShenandoahRendezvousClosure : public HandshakeClosure {
1885 public:
1886   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1887   inline void do_thread(Thread* thread) {}
1888 };
1889 
1890 void ShenandoahHeap::rendezvous_threads() {
1891   ShenandoahRendezvousClosure cl;
1892   Handshake::execute(&cl);
1893 }
1894 
1895 void ShenandoahHeap::recycle_trash() {
1896   free_set()->recycle_trash();
1897 }
1898 



































































































1899 void ShenandoahHeap::do_class_unloading() {
1900   _unloader.unload();
1901   if (mode()->is_generational()) {
1902     old_generation()->set_parseable(false);
1903   }
1904 }
1905 
1906 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1907   // Weak refs processing
1908   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1909                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1910   ShenandoahTimingsTracker t(phase);
1911   ShenandoahGCWorkerPhase worker_phase(phase);
1912   active_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
1913 }
1914 
1915 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1916   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1917 
1918   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1919   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1920   // for future GCLABs here.
1921   if (UseTLAB) {
1922     ShenandoahGCPhase phase(concurrent ?
1923                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1924                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1925     gclabs_retire(ResizeTLAB);
1926   }
1927 
1928   _update_refs_iterator.reset();
1929 }
1930 
1931 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1932   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1933   if (_gc_state_changed) {
1934     _gc_state_changed = false;
1935     char state = gc_state();
1936     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1937       ShenandoahThreadLocalData::set_gc_state(t, state);
1938     }
1939   }
1940 }
1941 
1942 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1943   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1944   _gc_state.set_cond(mask, value);
1945   _gc_state_changed = true;
1946 }
1947 
1948 void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
1949   uint mask;
1950   assert(!has_forwarded_objects(), "Young marking is not concurrent with evacuation");
1951   if (!in_progress && is_concurrent_old_mark_in_progress()) {
1952     assert(mode()->is_generational(), "Only generational GC has old marking");
1953     assert(_gc_state.is_set(MARKING), "concurrent_old_marking_in_progress implies MARKING");
1954     // If old-marking is in progress when we turn off YOUNG_MARKING, leave MARKING (and OLD_MARKING) on
1955     mask = YOUNG_MARKING;
1956   } else {
1957     mask = MARKING | YOUNG_MARKING;
1958   }
1959   set_gc_state(mask, in_progress);
1960   manage_satb_barrier(in_progress);
1961 }
1962 
1963 void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
1964 #ifdef ASSERT
1965   // has_forwarded_objects() iff UPDATEREFS or EVACUATION
1966   bool has_forwarded = has_forwarded_objects();
1967   bool updating_or_evacuating = _gc_state.is_set(UPDATEREFS | EVACUATION);
1968   bool evacuating = _gc_state.is_set(EVACUATION);
1969   assert ((has_forwarded == updating_or_evacuating) || (evacuating && !has_forwarded && collection_set()->is_empty()),
1970           "Updating or evacuating iff has forwarded objects, or if evacuation phase is promoting in place without forwarding");
1971 #endif
1972   if (!in_progress && is_concurrent_young_mark_in_progress()) {
1973     // If young-marking is in progress when we turn off OLD_MARKING, leave MARKING (and YOUNG_MARKING) on
1974     assert(_gc_state.is_set(MARKING), "concurrent_young_marking_in_progress implies MARKING");
1975     set_gc_state(OLD_MARKING, in_progress);
1976   } else {
1977     set_gc_state(MARKING | OLD_MARKING, in_progress);
1978   }
1979   manage_satb_barrier(in_progress);
1980 }
1981 
1982 bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const {
1983   return old_generation()->is_preparing_for_mark();
1984 }
1985 
1986 void ShenandoahHeap::manage_satb_barrier(bool active) {
1987   if (is_concurrent_mark_in_progress()) {
1988     // Ignore request to deactivate barrier while concurrent mark is in progress.
1989     // Do not attempt to re-activate the barrier if it is already active.
1990     if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
1991       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
1992     }
1993   } else {
1994     // No concurrent marking is in progress so honor request to deactivate,
1995     // but only if the barrier is already active.
1996     if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
1997       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
1998     }
1999   }
2000 }
2001 
2002 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2003   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2004   set_gc_state(EVACUATION, in_progress);
2005 }
2006 
2007 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2008   if (in_progress) {
2009     _concurrent_strong_root_in_progress.set();
2010   } else {
2011     _concurrent_strong_root_in_progress.unset();
2012   }
2013 }
2014 
2015 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
2016   set_gc_state(WEAK_ROOTS, cond);
2017 }
2018 
2019 GCTracer* ShenandoahHeap::tracer() {
2020   return shenandoah_policy()->tracer();
2021 }
2022 
2023 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2024   return _free_set->used();
2025 }
2026 
2027 bool ShenandoahHeap::try_cancel_gc() {
2028   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2029   return prev == CANCELLABLE;
2030 }
2031 
2032 void ShenandoahHeap::cancel_concurrent_mark() {
2033   if (mode()->is_generational()) {
2034     young_generation()->cancel_marking();
2035     old_generation()->cancel_marking();
2036   }
2037 
2038   global_generation()->cancel_marking();
2039 
2040   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
2041 }
2042 
2043 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2044   if (try_cancel_gc()) {
2045     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2046     log_info(gc)("%s", msg.buffer());
2047     Events::log(Thread::current(), "%s", msg.buffer());
2048     _cancel_requested_time = os::elapsedTime();
2049   }
2050 }
2051 
2052 uint ShenandoahHeap::max_workers() {
2053   return _max_workers;
2054 }
2055 
2056 void ShenandoahHeap::stop() {
2057   // The shutdown sequence should be able to terminate when GC is running.
2058 
2059   // Step 0. Notify policy to disable event recording and prevent visiting gc threads during shutdown
2060   _shenandoah_policy->record_shutdown();
2061 
2062   // Step 1. Notify control thread that we are in shutdown.
2063   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2064   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2065   control_thread()->prepare_for_graceful_shutdown();
2066 
2067   // Step 2. Notify GC workers that we are cancelling GC.
2068   cancel_gc(GCCause::_shenandoah_stop_vm);

2152 }
2153 
2154 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2155   set_gc_state(HAS_FORWARDED, cond);
2156 }
2157 
2158 void ShenandoahHeap::set_unload_classes(bool uc) {
2159   _unload_classes.set_cond(uc);
2160 }
2161 
2162 bool ShenandoahHeap::unload_classes() const {
2163   return _unload_classes.is_set();
2164 }
2165 
2166 address ShenandoahHeap::in_cset_fast_test_addr() {
2167   ShenandoahHeap* heap = ShenandoahHeap::heap();
2168   assert(heap->collection_set() != nullptr, "Sanity");
2169   return (address) heap->collection_set()->biased_map_address();
2170 }
2171 




2172 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2173   if (mode()->is_generational()) {
2174     young_generation()->reset_bytes_allocated_since_gc_start();
2175     old_generation()->reset_bytes_allocated_since_gc_start();
2176   }
2177 
2178   global_generation()->reset_bytes_allocated_since_gc_start();
2179 }
2180 
2181 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2182   _degenerated_gc_in_progress.set_cond(in_progress);
2183 }
2184 
2185 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2186   _full_gc_in_progress.set_cond(in_progress);
2187 }
2188 
2189 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2190   assert (is_full_gc_in_progress(), "should be");
2191   _full_gc_move_in_progress.set_cond(in_progress);
2192 }
2193 
2194 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2195   set_gc_state(UPDATEREFS, in_progress);
2196 }
2197 
2198 void ShenandoahHeap::register_nmethod(nmethod* nm) {

2222     if (r->is_active()) {
2223       if (r->is_pinned()) {
2224         if (r->pin_count() == 0) {
2225           r->make_unpinned();
2226         }
2227       } else {
2228         if (r->pin_count() > 0) {
2229           r->make_pinned();
2230         }
2231       }
2232     }
2233   }
2234 
2235   assert_pinned_region_status();
2236 }
2237 
2238 #ifdef ASSERT
2239 void ShenandoahHeap::assert_pinned_region_status() {
2240   for (size_t i = 0; i < num_regions(); i++) {
2241     ShenandoahHeapRegion* r = get_region(i);
2242     if (active_generation()->contains(r)) {
2243       assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2244              "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2245     }
2246   }
2247 }
2248 #endif
2249 
2250 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2251   return _gc_timer;
2252 }
2253 
2254 void ShenandoahHeap::prepare_concurrent_roots() {
2255   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2256   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2257   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2258   set_concurrent_weak_root_in_progress(true);
2259   if (unload_classes()) {
2260     _unloader.prepare();
2261   }
2262 }
2263 
2264 void ShenandoahHeap::finish_concurrent_roots() {
2265   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");

2280   } else {
2281     // Use ConcGCThreads outside safepoints
2282     assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2283            ConcGCThreads, nworkers);
2284   }
2285 }
2286 #endif
2287 
2288 ShenandoahVerifier* ShenandoahHeap::verifier() {
2289   guarantee(ShenandoahVerify, "Should be enabled");
2290   assert (_verifier != nullptr, "sanity");
2291   return _verifier;
2292 }
2293 
2294 template<bool CONCURRENT>
2295 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2296 private:
2297   ShenandoahHeap* _heap;
2298   ShenandoahRegionIterator* _regions;
2299 public:
2300   explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2301     WorkerTask("Shenandoah Update References"),
2302     _heap(ShenandoahHeap::heap()),
2303     _regions(regions) {
2304   }
2305 
2306   void work(uint worker_id) {
2307     if (CONCURRENT) {
2308       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2309       ShenandoahSuspendibleThreadSetJoiner stsj;
2310       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
2311     } else {
2312       ShenandoahParallelWorkerSession worker_session(worker_id);
2313       do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
2314     }
2315   }
2316 
2317 private:
2318   template<class T>
2319   void do_work(uint worker_id) {

2320     if (CONCURRENT && (worker_id == 0)) {
2321       // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
2322       // results of evacuation.  These reserves are no longer necessary because evacuation has completed.
2323       size_t cset_regions = _heap->collection_set()->count();
2324       // We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled because
2325       // we need the reclaimed collection set regions to replenish the collector reserves
2326       _heap->free_set()->move_collector_sets_to_mutator(cset_regions);
2327     }
2328     // If !CONCURRENT, there's no value in expanding Mutator free set
2329     T cl;
2330     ShenandoahHeapRegion* r = _regions->next();

2331     while (r != nullptr) {
2332       HeapWord* update_watermark = r->get_update_watermark();
2333       assert (update_watermark >= r->bottom(), "sanity");
2334       if (r->is_active() && !r->is_cset()) {
2335         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2336         if (ShenandoahPacing) {
2337           _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2338         }
2339       }
2340       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2341         return;
2342       }
2343       r = _regions->next();
2344     }
2345   }
2346 };
2347 
2348 void ShenandoahHeap::update_heap_references(bool concurrent) {
2349   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2350 
2351   if (concurrent) {
2352     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2353     workers()->run_task(&task);
2354   } else {
2355     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2356     workers()->run_task(&task);
2357   }
2358 }
2359 
2360 ShenandoahSynchronizePinnedRegionStates::ShenandoahSynchronizePinnedRegionStates() : _lock(ShenandoahHeap::heap()->lock()) { }
2361 
2362 void ShenandoahSynchronizePinnedRegionStates::heap_region_do(ShenandoahHeapRegion* r) {
2363   // Drop "pinned" state from regions that no longer have a pinned count. Put
2364   // regions with a pinned count into the "pinned" state.
2365   if (r->is_active()) {
2366     if (r->is_pinned()) {
2367       if (r->pin_count() == 0) {
2368         ShenandoahHeapLocker locker(_lock);
2369         r->make_unpinned();
2370       }
2371     } else {
2372       if (r->pin_count() > 0) {
2373         ShenandoahHeapLocker locker(_lock);
2374         r->make_pinned();









2375       }
2376     }
2377   }
2378 }


2379 
2380 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2381   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2382   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2383 
2384   {
2385     ShenandoahGCPhase phase(concurrent ?
2386                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2387                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2388 
2389     final_update_refs_update_region_states();
2390 
2391     assert_pinned_region_status();
2392   }
2393 
2394   {
2395     ShenandoahGCPhase phase(concurrent ?
2396                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2397                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2398     trash_cset_regions();
2399   }
2400 }
2401 
2402 void ShenandoahHeap::final_update_refs_update_region_states() {
2403   ShenandoahSynchronizePinnedRegionStates cl;
2404   parallel_heap_region_iterate(&cl);
2405 }
2406 
2407 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2408   ShenandoahGCPhase phase(concurrent ?
2409                           ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2410                           ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2411   ShenandoahHeapLocker locker(lock());
2412   size_t young_cset_regions, old_cset_regions;
2413   size_t first_old_region, last_old_region, old_region_count;
2414   _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
2415   // If there are no old regions, first_old_region will be greater than last_old_region
2416   assert((first_old_region > last_old_region) ||
2417          ((last_old_region + 1 - first_old_region >= old_region_count) &&
2418           get_region(first_old_region)->is_old() && get_region(last_old_region)->is_old()),
2419          "sanity: old_region_count: " SIZE_FORMAT ", first_old_region: " SIZE_FORMAT ", last_old_region: " SIZE_FORMAT,
2420          old_region_count, first_old_region, last_old_region);
2421 
2422   if (mode()->is_generational()) {
2423 #ifdef ASSERT
2424     if (ShenandoahVerify) {
2425       verifier()->verify_before_rebuilding_free_set();
2426     }
2427 #endif
2428 
2429     // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
2430     // available for transfer to old. Note that transfer of humongous regions does not impact available.
2431     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2432     size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
2433     gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
2434 
2435     // Total old_available may have been expanded to hold anticipated promotions.  We trigger if the fragmented available
2436     // memory represents more than 16 regions worth of data.  Note that fragmentation may increase when we promote regular
2437     // regions in place when many of these regular regions have an abundant amount of available memory within them.  Fragmentation
2438     // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
2439     //
2440     // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
2441     // within partially consumed regions of memory.
2442   }
2443   // Rebuild free set based on adjusted generation sizes.
2444   _free_set->rebuild(young_cset_regions, old_cset_regions);
2445 
2446   if (mode()->is_generational()) {
2447     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2448     ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
2449     old_gen->heuristics()->trigger_maybe(first_old_region, last_old_region, old_region_count, num_regions());
2450   }
2451 }
2452 
2453 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2454   print_on(st);
2455   st->cr();
2456   print_heap_regions_on(st);
2457 }
2458 
2459 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2460   size_t slice = r->index() / _bitmap_regions_per_slice;
2461 
2462   size_t regions_from = _bitmap_regions_per_slice * slice;
2463   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2464   for (size_t g = regions_from; g < regions_to; g++) {
2465     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2466     if (skip_self && g == r->index()) continue;
2467     if (get_region(g)->is_committed()) {
2468       return true;
2469     }

2552 void ShenandoahHeap::initialize_serviceability() {
2553   _memory_pool = new ShenandoahMemoryPool(this);
2554   _cycle_memory_manager.add_pool(_memory_pool);
2555   _stw_memory_manager.add_pool(_memory_pool);
2556 }
2557 
2558 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2559   GrowableArray<GCMemoryManager*> memory_managers(2);
2560   memory_managers.append(&_cycle_memory_manager);
2561   memory_managers.append(&_stw_memory_manager);
2562   return memory_managers;
2563 }
2564 
2565 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2566   GrowableArray<MemoryPool*> memory_pools(1);
2567   memory_pools.append(_memory_pool);
2568   return memory_pools;
2569 }
2570 
2571 MemoryUsage ShenandoahHeap::memory_usage() {
2572   return MemoryUsage(_initial_size, used(), committed(), max_capacity());
2573 }
2574 
2575 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2576   _heap(ShenandoahHeap::heap()),
2577   _index(0) {}
2578 
2579 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2580   _heap(heap),
2581   _index(0) {}
2582 
2583 void ShenandoahRegionIterator::reset() {
2584   _index = 0;
2585 }
2586 
2587 bool ShenandoahRegionIterator::has_next() const {
2588   return _index < _heap->num_regions();
2589 }
2590 
2591 char ShenandoahHeap::gc_state() const {
2592   return _gc_state.raw_value();

2617   }
2618 }
2619 
2620 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2621   if (is_idle()) return false;
2622 
2623   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2624   // marking phase.
2625   if (is_concurrent_mark_in_progress() &&
2626      !marking_context()->allocated_after_mark_start(obj)) {
2627     return true;
2628   }
2629 
2630   // Can not guarantee obj is deeply good.
2631   if (has_forwarded_objects()) {
2632     return true;
2633   }
2634 
2635   return false;
2636 }
2637 
2638 ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const {
2639   if (!mode()->is_generational()) {
2640     return global_generation();
2641   } else if (affiliation == YOUNG_GENERATION) {
2642     return young_generation();
2643   } else if (affiliation == OLD_GENERATION) {
2644     return old_generation();
2645   }
2646 
2647   ShouldNotReachHere();
2648   return nullptr;
2649 }
2650 
2651 void ShenandoahHeap::log_heap_status(const char* msg) const {
2652   if (mode()->is_generational()) {
2653     young_generation()->log_status(msg);
2654     old_generation()->log_status(msg);
2655   } else {
2656     global_generation()->log_status(msg);
2657   }
2658 }
< prev index next >