1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/oopStorageSet.hpp"
  35 #include "gc/shared/plab.hpp"
  36 
  37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  43 #include "gc/shenandoah/shenandoahControlThread.hpp"
  44 #include "gc/shenandoah/shenandoahEvacLockingBitmap.hpp"
  45 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  46 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  47 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  48 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  49 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  50 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  51 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  52 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  53 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  54 #include "gc/shenandoah/shenandoahMetrics.hpp"
  55 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  61 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  62 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  63 #include "gc/shenandoah/shenandoahUtils.hpp"
  64 #include "gc/shenandoah/shenandoahVerifier.hpp"
  65 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  66 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  67 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  68 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  69 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  70 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  71 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  72 #if INCLUDE_JFR
  73 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  74 #endif
  75 
  76 #include "memory/metaspace.hpp"
  77 #include "oops/compressedOops.inline.hpp"
  78 #include "runtime/atomic.hpp"
  79 #include "runtime/globals.hpp"
  80 #include "runtime/interfaceSupport.inline.hpp"
  81 #include "runtime/orderAccess.hpp"
  82 #include "runtime/safepointMechanism.hpp"
  83 #include "runtime/vmThread.hpp"
  84 #include "services/mallocTracker.hpp"
  85 #include "utilities/powerOfTwo.hpp"
  86 
  87 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
  88 
  89 #ifdef ASSERT
  90 template <class T>
  91 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  92   T o = RawAccess<>::oop_load(p);
  93   if (! CompressedOops::is_null(o)) {
  94     oop obj = CompressedOops::decode_not_null(o);
  95     shenandoah_assert_not_forwarded(p, obj);
  96   }
  97 }
  98 
  99 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
 100 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
 101 #endif
 102 
 103 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 104 private:
 105   ShenandoahRegionIterator _regions;
 106   const size_t _page_size;
 107 public:
 108   ShenandoahPretouchHeapTask(size_t page_size) :
 109     AbstractGangTask("Shenandoah Pretouch Heap"),
 110     _page_size(page_size) {}
 111 
 112   virtual void work(uint worker_id) {
 113     ShenandoahHeapRegion* r = _regions.next();
 114     while (r != NULL) {
 115       if (r->is_committed()) {
 116         os::pretouch_memory(r->bottom(), r->end(), _page_size);
 117       }
 118       r = _regions.next();
 119     }
 120   }
 121 };
 122 
 123 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 124 private:
 125   ShenandoahRegionIterator _regions;
 126   char* _bitmap_base;
 127   const size_t _bitmap_size;
 128   const size_t _page_size;
 129 public:
 130   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 131     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 132     _bitmap_base(bitmap_base),
 133     _bitmap_size(bitmap_size),
 134     _page_size(page_size) {}
 135 
 136   virtual void work(uint worker_id) {
 137     ShenandoahHeapRegion* r = _regions.next();
 138     while (r != NULL) {
 139       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 140       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 141       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 142 
 143       if (r->is_committed()) {
 144         os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 145       }
 146 
 147       r = _regions.next();
 148     }
 149   }
 150 };
 151 
 152 jint ShenandoahHeap::initialize() {
 153   //
 154   // Figure out heap sizing
 155   //
 156 
 157   size_t init_byte_size = InitialHeapSize;
 158   size_t min_byte_size  = MinHeapSize;
 159   size_t max_byte_size  = MaxHeapSize;
 160   size_t heap_alignment = HeapAlignment;
 161 
 162   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 163 
 164   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 165   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 166 
 167   _num_regions = ShenandoahHeapRegion::region_count();
 168 
 169   // Now we know the number of regions, initialize the heuristics.
 170   initialize_heuristics();
 171 
 172   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 173   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 174   assert(num_committed_regions <= _num_regions, "sanity");
 175   _initial_size = num_committed_regions * reg_size_bytes;
 176 
 177   size_t num_min_regions = min_byte_size / reg_size_bytes;
 178   num_min_regions = MIN2(num_min_regions, _num_regions);
 179   assert(num_min_regions <= _num_regions, "sanity");
 180   _minimum_size = num_min_regions * reg_size_bytes;
 181 
 182   _committed = _initial_size;
 183 
 184   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 185   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 186   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 187 
 188   //
 189   // Reserve and commit memory for heap
 190   //
 191 
 192   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 193   initialize_reserved_region(heap_rs);
 194   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 195   _heap_region_special = heap_rs.special();
 196 
 197   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 198          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 199 
 200 #if SHENANDOAH_OPTIMIZED_OBJTASK
 201   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 202   // Fail if we ever attempt to address more than we can.
 203   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 204     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 205                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 206                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 207                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 208     vm_exit_during_initialization("Fatal Error", buf);
 209   }
 210 #endif
 211 
 212   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 213   if (!_heap_region_special) {
 214     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 215                               "Cannot commit heap memory");
 216   }
 217 
 218   //
 219   // Reserve and commit memory for bitmap(s)
 220   //
 221 
 222   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 223   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 224 
 225   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 226 
 227   guarantee(bitmap_bytes_per_region != 0,
 228             "Bitmap bytes per region should not be zero");
 229   guarantee(is_power_of_2(bitmap_bytes_per_region),
 230             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 231 
 232   if (bitmap_page_size > bitmap_bytes_per_region) {
 233     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 234     _bitmap_bytes_per_slice = bitmap_page_size;
 235   } else {
 236     _bitmap_regions_per_slice = 1;
 237     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 238   }
 239 
 240   guarantee(_bitmap_regions_per_slice >= 1,
 241             "Should have at least one region per slice: " SIZE_FORMAT,
 242             _bitmap_regions_per_slice);
 243 
 244   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 245             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 246             _bitmap_bytes_per_slice, bitmap_page_size);
 247 
 248   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 249   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 250   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 251   _bitmap_region_special = bitmap.special();
 252 
 253   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 254                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 255   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 256   if (!_bitmap_region_special) {
 257     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 258                               "Cannot commit bitmap memory");
 259   }
 260 
 261   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 262 
 263   _evac_locking_bitmap = new ShenandoahEvacLockingBitmap(_heap_region);
 264 
 265   if (ShenandoahVerify) {
 266     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 267     if (!verify_bitmap.special()) {
 268       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 269                                 "Cannot commit verification bitmap memory");
 270     }
 271     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 272     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 273     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 274     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 275   }
 276 
 277   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 278   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 279   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 280   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 281   _aux_bitmap_region_special = aux_bitmap.special();
 282   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 283 
 284   //
 285   // Create regions and region sets
 286   //
 287   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 288   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 289   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 290 
 291   ReservedSpace region_storage(region_storage_size, region_page_size);
 292   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 293   if (!region_storage.special()) {
 294     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 295                               "Cannot commit region memory");
 296   }
 297 
 298   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 299   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 300   // If not successful, bite a bullet and allocate at whatever address.
 301   {
 302     size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 303     size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 304 
 305     uintptr_t min = round_up_power_of_2(cset_align);
 306     uintptr_t max = (1u << 30u);
 307 
 308     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 309       char* req_addr = (char*)addr;
 310       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 311       ReservedSpace cset_rs(cset_size, cset_align, false, req_addr);
 312       if (cset_rs.is_reserved()) {
 313         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 314         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 315         break;
 316       }
 317     }
 318 
 319     if (_collection_set == NULL) {
 320       ReservedSpace cset_rs(cset_size, cset_align, false);
 321       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 322     }
 323   }
 324 
 325   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 326   _free_set = new ShenandoahFreeSet(this, _num_regions);
 327 
 328   {
 329     ShenandoahHeapLocker locker(lock());
 330 
 331     for (size_t i = 0; i < _num_regions; i++) {
 332       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 333       bool is_committed = i < num_committed_regions;
 334       void* loc = region_storage.base() + i * region_align;
 335 
 336       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 337       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 338 
 339       _marking_context->initialize_top_at_mark_start(r);
 340       _regions[i] = r;
 341       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 342     }
 343 
 344     // Initialize to complete
 345     _marking_context->mark_complete();
 346 
 347     _free_set->rebuild();
 348   }
 349 
 350   if (AlwaysPreTouch) {
 351     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 352     // before initialize() below zeroes it with initializing thread. For any given region,
 353     // we touch the region and the corresponding bitmaps from the same thread.
 354     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 355 
 356     _pretouch_heap_page_size = heap_page_size;
 357     _pretouch_bitmap_page_size = bitmap_page_size;
 358 
 359 #ifdef LINUX
 360     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 361     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 362     // them into huge one. Therefore, we need to pretouch with smaller pages.
 363     if (UseTransparentHugePages) {
 364       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 365       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 366     }
 367 #endif
 368 
 369     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 370     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 371 
 372     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 373     _workers->run_task(&bcl);
 374 
 375     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 376     _workers->run_task(&hcl);
 377   }
 378 
 379   //
 380   // Initialize the rest of GC subsystems
 381   //
 382 
 383   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 384   for (uint worker = 0; worker < _max_workers; worker++) {
 385     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 386     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 387   }
 388 
 389   // There should probably be Shenandoah-specific options for these,
 390   // just as there are G1-specific options.
 391   {
 392     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 393     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 394     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 395   }
 396 
 397   _monitoring_support = new ShenandoahMonitoringSupport(this);
 398   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 399   ShenandoahStringDedup::initialize();
 400   ShenandoahCodeRoots::initialize();
 401 
 402   if (ShenandoahPacing) {
 403     _pacer = new ShenandoahPacer(this);
 404     _pacer->setup_for_idle();
 405   } else {
 406     _pacer = NULL;
 407   }
 408 
 409   _control_thread = new ShenandoahControlThread();
 410 
 411   _ref_proc_mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
 412   _ref_proc_mt_discovery = _max_workers > 1;
 413 
 414   ShenandoahInitLogger::print();
 415 
 416   return JNI_OK;
 417 }
 418 
 419 void ShenandoahHeap::initialize_heuristics() {
 420   if (ShenandoahGCMode != NULL) {
 421     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 422       _gc_mode = new ShenandoahSATBMode();
 423     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 424       _gc_mode = new ShenandoahIUMode();
 425     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 426       _gc_mode = new ShenandoahPassiveMode();
 427     } else {
 428       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 429     }
 430   } else {
 431     ShouldNotReachHere();
 432   }
 433   _gc_mode->initialize_flags();
 434   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 435     vm_exit_during_initialization(
 436             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 437                     _gc_mode->name()));
 438   }
 439   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 440     vm_exit_during_initialization(
 441             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 442                     _gc_mode->name()));
 443   }
 444 
 445   _heuristics = _gc_mode->initialize_heuristics();
 446 
 447   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 448     vm_exit_during_initialization(
 449             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 450                     _heuristics->name()));
 451   }
 452   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 453     vm_exit_during_initialization(
 454             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 455                     _heuristics->name()));
 456   }
 457 }
 458 
 459 #ifdef _MSC_VER
 460 #pragma warning( push )
 461 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 462 #endif
 463 
 464 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 465   CollectedHeap(),
 466   _initial_size(0),
 467   _used(0),
 468   _committed(0),
 469   _bytes_allocated_since_gc_start(0),
 470   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 471   _workers(NULL),
 472   _safepoint_workers(NULL),
 473   _heap_region_special(false),
 474   _num_regions(0),
 475   _regions(NULL),
 476   _update_refs_iterator(this),
 477   _control_thread(NULL),
 478   _shenandoah_policy(policy),
 479   _heuristics(NULL),
 480   _free_set(NULL),
 481   _scm(new ShenandoahConcurrentMark()),
 482   _full_gc(new ShenandoahMarkCompact()),
 483   _pacer(NULL),
 484   _verifier(NULL),
 485   _phase_timings(NULL),
 486   _monitoring_support(NULL),
 487   _memory_pool(NULL),
 488   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 489   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 490   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 491   _soft_ref_policy(),
 492   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 493   _ref_processor(NULL),
 494   _marking_context(NULL),
 495   _bitmap_size(0),
 496   _bitmap_regions_per_slice(0),
 497   _bitmap_bytes_per_slice(0),
 498   _bitmap_region_special(false),
 499   _aux_bitmap_region_special(false),
 500   _liveness_cache(NULL),
 501   _collection_set(NULL),
 502   _evac_locking_bitmap(NULL)
 503 {
 504   _heap = this;
 505 
 506   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 507 
 508   _max_workers = MAX2(_max_workers, 1U);
 509   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 510                             /* are_GC_task_threads */ true,
 511                             /* are_ConcurrentGC_threads */ true);
 512   if (_workers == NULL) {
 513     vm_exit_during_initialization("Failed necessary allocation.");
 514   } else {
 515     _workers->initialize_workers();
 516   }
 517 
 518   if (ParallelGCThreads > 1) {
 519     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 520                                                 ParallelGCThreads,
 521                       /* are_GC_task_threads */ false,
 522                  /* are_ConcurrentGC_threads */ false);
 523     _safepoint_workers->initialize_workers();
 524   }
 525 }
 526 
 527 #ifdef _MSC_VER
 528 #pragma warning( pop )
 529 #endif
 530 
 531 class ShenandoahResetBitmapTask : public AbstractGangTask {
 532 private:
 533   ShenandoahRegionIterator _regions;
 534 
 535 public:
 536   ShenandoahResetBitmapTask() :
 537     AbstractGangTask("Parallel Reset Bitmap Task") {}
 538 
 539   void work(uint worker_id) {
 540     ShenandoahHeapRegion* region = _regions.next();
 541     ShenandoahHeap* heap = ShenandoahHeap::heap();
 542     ShenandoahMarkingContext* const ctx = heap->marking_context();
 543     while (region != NULL) {
 544       if (heap->is_bitmap_slice_committed(region)) {
 545         ctx->clear_bitmap(region);
 546       }
 547       region = _regions.next();
 548     }
 549   }
 550 };
 551 
 552 void ShenandoahHeap::reset_mark_bitmap() {
 553   assert_gc_workers(_workers->active_workers());
 554   mark_incomplete_marking_context();
 555 
 556   ShenandoahResetBitmapTask task;
 557   _workers->run_task(&task);
 558 }
 559 
 560 void ShenandoahHeap::print_on(outputStream* st) const {
 561   st->print_cr("Shenandoah Heap");
 562   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 563                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 564                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 565                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 566   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 567                num_regions(),
 568                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 569                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 570 
 571   st->print("Status: ");
 572   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 573   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 574   if (is_evacuation_in_progress())             st->print("evacuating, ");
 575   if (is_update_refs_in_progress())            st->print("updating refs, ");
 576   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 577   if (is_full_gc_in_progress())                st->print("full gc, ");
 578   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 579   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 580   if (is_concurrent_strong_root_in_progress() &&
 581       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 582 
 583   if (cancelled_gc()) {
 584     st->print("cancelled");
 585   } else {
 586     st->print("not cancelled");
 587   }
 588   st->cr();
 589 
 590   st->print_cr("Reserved region:");
 591   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 592                p2i(reserved_region().start()),
 593                p2i(reserved_region().end()));
 594 
 595   ShenandoahCollectionSet* cset = collection_set();
 596   st->print_cr("Collection set:");
 597   if (cset != NULL) {
 598     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 599     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 600   } else {
 601     st->print_cr(" (NULL)");
 602   }
 603 
 604   st->cr();
 605   MetaspaceUtils::print_on(st);
 606 
 607   if (Verbose) {
 608     print_heap_regions_on(st);
 609   }
 610 }
 611 
 612 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 613 public:
 614   void do_thread(Thread* thread) {
 615     assert(thread != NULL, "Sanity");
 616     assert(thread->is_Worker_thread(), "Only worker thread expected");
 617     ShenandoahThreadLocalData::initialize_gclab(thread);
 618   }
 619 };
 620 
 621 void ShenandoahHeap::post_initialize() {
 622   CollectedHeap::post_initialize();
 623   MutexLocker ml(Threads_lock);
 624 
 625   ShenandoahInitWorkerGCLABClosure init_gclabs;
 626   _workers->threads_do(&init_gclabs);
 627 
 628   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 629   // Now, we will let WorkGang to initialize gclab when new worker is created.
 630   _workers->set_initialize_gclab();
 631 
 632   _scm->initialize(_max_workers);
 633   _full_gc->initialize(_gc_timer);
 634 
 635   ref_processing_init();
 636 
 637   _heuristics->initialize();
 638 
 639   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 640 }
 641 
 642 size_t ShenandoahHeap::used() const {
 643   return Atomic::load_acquire(&_used);
 644 }
 645 
 646 size_t ShenandoahHeap::committed() const {
 647   OrderAccess::acquire();
 648   return _committed;
 649 }
 650 
 651 void ShenandoahHeap::increase_committed(size_t bytes) {
 652   shenandoah_assert_heaplocked_or_safepoint();
 653   _committed += bytes;
 654 }
 655 
 656 void ShenandoahHeap::decrease_committed(size_t bytes) {
 657   shenandoah_assert_heaplocked_or_safepoint();
 658   _committed -= bytes;
 659 }
 660 
 661 void ShenandoahHeap::increase_used(size_t bytes) {
 662   Atomic::add(&_used, bytes);
 663 }
 664 
 665 void ShenandoahHeap::set_used(size_t bytes) {
 666   Atomic::release_store_fence(&_used, bytes);
 667 }
 668 
 669 void ShenandoahHeap::decrease_used(size_t bytes) {
 670   assert(used() >= bytes, "never decrease heap size by more than we've left");
 671   Atomic::sub(&_used, bytes);
 672 }
 673 
 674 void ShenandoahHeap::increase_allocated(size_t bytes) {
 675   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 676 }
 677 
 678 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 679   size_t bytes = words * HeapWordSize;
 680   if (!waste) {
 681     increase_used(bytes);
 682   }
 683   increase_allocated(bytes);
 684   if (ShenandoahPacing) {
 685     control_thread()->pacing_notify_alloc(words);
 686     if (waste) {
 687       pacer()->claim_for_alloc(words, true);
 688     }
 689   }
 690 }
 691 
 692 size_t ShenandoahHeap::capacity() const {
 693   return committed();
 694 }
 695 
 696 size_t ShenandoahHeap::max_capacity() const {
 697   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 698 }
 699 
 700 size_t ShenandoahHeap::min_capacity() const {
 701   return _minimum_size;
 702 }
 703 
 704 size_t ShenandoahHeap::initial_capacity() const {
 705   return _initial_size;
 706 }
 707 
 708 bool ShenandoahHeap::is_in(const void* p) const {
 709   HeapWord* heap_base = (HeapWord*) base();
 710   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 711   return p >= heap_base && p < last_region_end;
 712 }
 713 
 714 void ShenandoahHeap::op_uncommit(double shrink_before) {
 715   assert (ShenandoahUncommit, "should be enabled");
 716 
 717   // Application allocates from the beginning of the heap, and GC allocates at
 718   // the end of it. It is more efficient to uncommit from the end, so that applications
 719   // could enjoy the near committed regions. GC allocations are much less frequent,
 720   // and therefore can accept the committing costs.
 721 
 722   size_t count = 0;
 723   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 724     ShenandoahHeapRegion* r = get_region(i - 1);
 725     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 726       ShenandoahHeapLocker locker(lock());
 727       if (r->is_empty_committed()) {
 728         // Do not uncommit below minimal capacity
 729         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 730           break;
 731         }
 732 
 733         r->make_uncommitted();
 734         count++;
 735       }
 736     }
 737     SpinPause(); // allow allocators to take the lock
 738   }
 739 
 740   if (count > 0) {
 741     control_thread()->notify_heap_changed();
 742   }
 743 }
 744 
 745 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 746   // New object should fit the GCLAB size
 747   size_t min_size = MAX2(size, PLAB::min_size());
 748 
 749   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 750   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 751   new_size = MIN2(new_size, PLAB::max_size());
 752   new_size = MAX2(new_size, PLAB::min_size());
 753 
 754   // Record new heuristic value even if we take any shortcut. This captures
 755   // the case when moderately-sized objects always take a shortcut. At some point,
 756   // heuristics should catch up with them.
 757   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 758 
 759   if (new_size < size) {
 760     // New size still does not fit the object. Fall back to shared allocation.
 761     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 762     return NULL;
 763   }
 764 
 765   // Retire current GCLAB, and allocate a new one.
 766   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 767   gclab->retire();
 768 
 769   size_t actual_size = 0;
 770   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 771   if (gclab_buf == NULL) {
 772     return NULL;
 773   }
 774 
 775   assert (size <= actual_size, "allocation should fit");
 776 
 777   if (ZeroTLAB) {
 778     // ..and clear it.
 779     Copy::zero_to_words(gclab_buf, actual_size);
 780   } else {
 781     // ...and zap just allocated object.
 782 #ifdef ASSERT
 783     // Skip mangling the space corresponding to the object header to
 784     // ensure that the returned space is not considered parsable by
 785     // any concurrent GC thread.
 786     size_t hdr_size = oopDesc::header_size();
 787     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 788 #endif // ASSERT
 789   }
 790   gclab->set_buf(gclab_buf, actual_size);
 791   return gclab->allocate(size);
 792 }
 793 
 794 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 795                                             size_t requested_size,
 796                                             size_t* actual_size) {
 797   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 798   HeapWord* res = allocate_memory(req);
 799   if (res != NULL) {
 800     *actual_size = req.actual_size();
 801   } else {
 802     *actual_size = 0;
 803   }
 804   return res;
 805 }
 806 
 807 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 808                                              size_t word_size,
 809                                              size_t* actual_size) {
 810   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 811   HeapWord* res = allocate_memory(req);
 812   if (res != NULL) {
 813     *actual_size = req.actual_size();
 814   } else {
 815     *actual_size = 0;
 816   }
 817   return res;
 818 }
 819 
 820 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 821   intptr_t pacer_epoch = 0;
 822   bool in_new_region = false;
 823   HeapWord* result = NULL;
 824 
 825   if (req.is_mutator_alloc()) {
 826     if (ShenandoahPacing) {
 827       pacer()->pace_for_alloc(req.size());
 828       pacer_epoch = pacer()->epoch();
 829     }
 830 
 831     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 832       result = allocate_memory_under_lock(req, in_new_region);
 833     }
 834 
 835     // Allocation failed, block until control thread reacted, then retry allocation.
 836     //
 837     // It might happen that one of the threads requesting allocation would unblock
 838     // way later after GC happened, only to fail the second allocation, because
 839     // other threads have already depleted the free storage. In this case, a better
 840     // strategy is to try again, as long as GC makes progress.
 841     //
 842     // Then, we need to make sure the allocation was retried after at least one
 843     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 844 
 845     size_t tries = 0;
 846 
 847     while (result == NULL && _progress_last_gc.is_set()) {
 848       tries++;
 849       control_thread()->handle_alloc_failure(req);
 850       result = allocate_memory_under_lock(req, in_new_region);
 851     }
 852 
 853     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 854       tries++;
 855       control_thread()->handle_alloc_failure(req);
 856       result = allocate_memory_under_lock(req, in_new_region);
 857     }
 858 
 859   } else {
 860     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 861     result = allocate_memory_under_lock(req, in_new_region);
 862     // Do not call handle_alloc_failure() here, because we cannot block.
 863     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 864   }
 865 
 866   if (in_new_region) {
 867     control_thread()->notify_heap_changed();
 868   }
 869 
 870   if (result != NULL) {
 871     size_t requested = req.size();
 872     size_t actual = req.actual_size();
 873 
 874     assert (req.is_lab_alloc() || (requested == actual),
 875             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 876             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 877 
 878     if (req.is_mutator_alloc()) {
 879       notify_mutator_alloc_words(actual, false);
 880 
 881       // If we requested more than we were granted, give the rest back to pacer.
 882       // This only matters if we are in the same pacing epoch: do not try to unpace
 883       // over the budget for the other phase.
 884       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 885         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 886       }
 887     } else {
 888       increase_used(actual*HeapWordSize);
 889     }
 890   }
 891 
 892   return result;
 893 }
 894 
 895 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 896   ShenandoahHeapLocker locker(lock());
 897   return _free_set->allocate(req, in_new_region);
 898 }
 899 
 900 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 901                                         bool*  gc_overhead_limit_was_exceeded) {
 902   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 903   return allocate_memory(req);
 904 }
 905 
 906 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 907                                                              size_t size,
 908                                                              Metaspace::MetadataType mdtype) {
 909   MetaWord* result;
 910 
 911   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 912   if (heuristics()->can_unload_classes()) {
 913     ShenandoahHeuristics* h = heuristics();
 914     h->record_metaspace_oom();
 915   }
 916 
 917   // Expand and retry allocation
 918   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 919   if (result != NULL) {
 920     return result;
 921   }
 922 
 923   // Start full GC
 924   collect(GCCause::_metadata_GC_clear_soft_refs);
 925 
 926   // Retry allocation
 927   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 928   if (result != NULL) {
 929     return result;
 930   }
 931 
 932   // Expand and retry allocation
 933   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 934   if (result != NULL) {
 935     return result;
 936   }
 937 
 938   // Out of memory
 939   return NULL;
 940 }
 941 
 942 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 943 private:
 944   ShenandoahHeap* const _heap;
 945   Thread* const _thread;
 946 public:
 947   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 948     _heap(heap), _thread(Thread::current()) {}
 949 
 950   void do_object(oop p) {
 951     shenandoah_assert_marked(NULL, p);
 952     if (!p->is_forwarded()) {
 953       _heap->evacuate_object(p, _thread);
 954     }
 955   }
 956 };
 957 
 958 class ShenandoahEvacuationTask : public AbstractGangTask {
 959 private:
 960   ShenandoahHeap* const _sh;
 961   ShenandoahCollectionSet* const _cs;
 962   bool _concurrent;
 963 public:
 964   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 965                            ShenandoahCollectionSet* cs,
 966                            bool concurrent) :
 967     AbstractGangTask("Parallel Evacuation Task"),
 968     _sh(sh),
 969     _cs(cs),
 970     _concurrent(concurrent)
 971   {}
 972 
 973   void work(uint worker_id) {
 974     if (_concurrent) {
 975       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 976       SuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 977       do_work();
 978     } else {
 979       ShenandoahParallelWorkerSession worker_session(worker_id);
 980       do_work();
 981     }
 982   }
 983 
 984 private:
 985   void do_work() {
 986     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 987     ShenandoahHeapRegion* r;
 988     while ((r =_cs->claim_next()) != NULL) {
 989       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 990       _sh->marked_object_iterate(r, &cl);
 991 
 992       if (ShenandoahPacing) {
 993         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 994       }
 995 
 996       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 997         break;
 998       }
 999     }
1000   }
1001 };
1002 
1003 void ShenandoahHeap::trash_cset_regions() {
1004   ShenandoahHeapLocker locker(lock());
1005 
1006   ShenandoahCollectionSet* set = collection_set();
1007   ShenandoahHeapRegion* r;
1008   set->clear_current_index();
1009   while ((r = set->next()) != NULL) {
1010     r->make_trash();
1011   }
1012   collection_set()->clear();
1013 }
1014 
1015 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1016   st->print_cr("Heap Regions:");
1017   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1018   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1019   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1020   st->print_cr("SN=alloc sequence number");
1021 
1022   for (size_t i = 0; i < num_regions(); i++) {
1023     get_region(i)->print_on(st);
1024   }
1025 }
1026 
1027 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1028   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1029 
1030   oop humongous_obj = oop(start->bottom());
1031   size_t size = humongous_obj->size();
1032   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1033   size_t index = start->index() + required_regions - 1;
1034 
1035   assert(!start->has_live(), "liveness must be zero");
1036 
1037   for(size_t i = 0; i < required_regions; i++) {
1038     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1039     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1040     ShenandoahHeapRegion* region = get_region(index --);
1041 
1042     assert(region->is_humongous(), "expect correct humongous start or continuation");
1043     assert(!region->is_cset(), "Humongous region should not be in collection set");
1044 
1045     region->make_trash_immediate();
1046   }
1047 }
1048 
1049 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1050 public:
1051   void do_thread(Thread* thread) {
1052     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1053     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1054     gclab->retire();
1055   }
1056 };
1057 
1058 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1059   if (UseTLAB) {
1060     CollectedHeap::ensure_parsability(retire_tlabs);
1061   }
1062   ShenandoahRetireGCLABClosure cl;
1063   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1064     cl.do_thread(t);
1065   }
1066   workers()->threads_do(&cl);
1067 }
1068 
1069 void ShenandoahHeap::resize_tlabs() {
1070   CollectedHeap::resize_all_tlabs();
1071 }
1072 
1073 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1074 private:
1075   ShenandoahRootEvacuator* _rp;
1076 
1077 public:
1078   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1079     AbstractGangTask("Shenandoah evacuate and update roots"),
1080     _rp(rp) {}
1081 
1082   void work(uint worker_id) {
1083     ShenandoahParallelWorkerSession worker_session(worker_id);
1084     ShenandoahEvacuateUpdateRootsClosure<> cl;
1085     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1086     _rp->roots_do(worker_id, &cl);
1087   }
1088 };
1089 
1090 void ShenandoahHeap::evacuate_and_update_roots() {
1091 #if COMPILER2_OR_JVMCI
1092   DerivedPointerTable::clear();
1093 #endif
1094   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1095   {
1096     // Include concurrent roots if current cycle can not process those roots concurrently
1097     ShenandoahRootEvacuator rp(workers()->active_workers(),
1098                                ShenandoahPhaseTimings::init_evac,
1099                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1100                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1101     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1102     workers()->run_task(&roots_task);
1103   }
1104 
1105 #if COMPILER2_OR_JVMCI
1106   DerivedPointerTable::update_pointers();
1107 #endif
1108 }
1109 
1110 // Returns size in bytes
1111 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1112   if (ShenandoahElasticTLAB) {
1113     // With Elastic TLABs, return the max allowed size, and let the allocation path
1114     // figure out the safe size for current allocation.
1115     return ShenandoahHeapRegion::max_tlab_size_bytes();
1116   } else {
1117     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1118   }
1119 }
1120 
1121 size_t ShenandoahHeap::max_tlab_size() const {
1122   // Returns size in words
1123   return ShenandoahHeapRegion::max_tlab_size_words();
1124 }
1125 
1126 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1127 public:
1128   void do_thread(Thread* thread) {
1129     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1130     gclab->retire();
1131     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1132       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1133     }
1134   }
1135 };
1136 
1137 void ShenandoahHeap::retire_and_reset_gclabs() {
1138   ShenandoahRetireAndResetGCLABClosure cl;
1139   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1140     cl.do_thread(t);
1141   }
1142   workers()->threads_do(&cl);
1143 }
1144 
1145 void ShenandoahHeap::collect(GCCause::Cause cause) {
1146   control_thread()->request_gc(cause);
1147 }
1148 
1149 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1150   //assert(false, "Shouldn't need to do full collections");
1151 }
1152 
1153 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1154   ShenandoahHeapRegion* r = heap_region_containing(addr);
1155   if (r != NULL) {
1156     return r->block_start(addr);
1157   }
1158   return NULL;
1159 }
1160 
1161 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1162   ShenandoahHeapRegion* r = heap_region_containing(addr);
1163   return r->block_is_obj(addr);
1164 }
1165 
1166 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1167   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1168 }
1169 
1170 jlong ShenandoahHeap::millis_since_last_gc() {
1171   double v = heuristics()->time_since_last_gc() * 1000;
1172   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1173   return (jlong)v;
1174 }
1175 
1176 void ShenandoahHeap::prepare_for_verify() {
1177   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1178     make_parsable(false);
1179   }
1180 }
1181 
1182 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1183   workers()->threads_do(tcl);
1184   if (_safepoint_workers != NULL) {
1185     _safepoint_workers->threads_do(tcl);
1186   }
1187   if (ShenandoahStringDedup::is_enabled()) {
1188     ShenandoahStringDedup::threads_do(tcl);
1189   }
1190 }
1191 
1192 void ShenandoahHeap::print_tracing_info() const {
1193   LogTarget(Info, gc, stats) lt;
1194   if (lt.is_enabled()) {
1195     ResourceMark rm;
1196     LogStream ls(lt);
1197 
1198     phase_timings()->print_global_on(&ls);
1199 
1200     ls.cr();
1201     ls.cr();
1202 
1203     shenandoah_policy()->print_gc_stats(&ls);
1204 
1205     ls.cr();
1206     ls.cr();
1207 
1208     if (ShenandoahPacing) {
1209       pacer()->print_on(&ls);
1210     }
1211 
1212     ls.cr();
1213     ls.cr();
1214   }
1215 }
1216 
1217 void ShenandoahHeap::verify(VerifyOption vo) {
1218   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1219     if (ShenandoahVerify) {
1220       verifier()->verify_generic(vo);
1221     } else {
1222       // TODO: Consider allocating verification bitmaps on demand,
1223       // and turn this on unconditionally.
1224     }
1225   }
1226 }
1227 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1228   return _free_set->capacity();
1229 }
1230 
1231 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1232 private:
1233   MarkBitMap* _bitmap;
1234   Stack<oop,mtGC>* _oop_stack;
1235   ShenandoahHeap* const _heap;
1236   ShenandoahMarkingContext* const _marking_context;
1237 
1238   template <class T>
1239   void do_oop_work(T* p) {
1240     T o = RawAccess<>::oop_load(p);
1241     if (!CompressedOops::is_null(o)) {
1242       oop obj = CompressedOops::decode_not_null(o);
1243       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1244         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1245         return;
1246       }
1247       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1248 
1249       assert(oopDesc::is_oop(obj), "must be a valid oop");
1250       if (!_bitmap->is_marked(obj)) {
1251         _bitmap->mark(obj);
1252         _oop_stack->push(obj);
1253       }
1254     }
1255   }
1256 public:
1257   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1258     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1259     _marking_context(_heap->marking_context()) {}
1260   void do_oop(oop* p)       { do_oop_work(p); }
1261   void do_oop(narrowOop* p) { do_oop_work(p); }
1262 };
1263 
1264 /*
1265  * This is public API, used in preparation of object_iterate().
1266  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1267  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1268  * control, we call SH::make_tlabs_parsable().
1269  */
1270 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1271   // No-op.
1272 }
1273 
1274 /*
1275  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1276  *
1277  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1278  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1279  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1280  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1281  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1282  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1283  * wiped the bitmap in preparation for next marking).
1284  *
1285  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1286  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1287  * is allowed to report dead objects, but is not required to do so.
1288  */
1289 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1290   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1291   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1292     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1293     return;
1294   }
1295 
1296   // Reset bitmap
1297   _aux_bit_map.clear();
1298 
1299   Stack<oop,mtGC> oop_stack;
1300 
1301   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1302 
1303   {
1304     // First, we process GC roots according to current GC cycle.
1305     // This populates the work stack with initial objects.
1306     // It is important to relinquish the associated locks before diving
1307     // into heap dumper.
1308     ShenandoahHeapIterationRootScanner rp;
1309     rp.roots_do(&oops);
1310   }
1311 
1312   // Work through the oop stack to traverse heap.
1313   while (! oop_stack.is_empty()) {
1314     oop obj = oop_stack.pop();
1315     assert(oopDesc::is_oop(obj), "must be a valid oop");
1316     cl->do_object(obj);
1317     obj->oop_iterate(&oops);
1318   }
1319 
1320   assert(oop_stack.is_empty(), "should be empty");
1321 
1322   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1323     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1324   }
1325 }
1326 
1327 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1328 void ShenandoahHeap::keep_alive(oop obj) {
1329   if (is_concurrent_mark_in_progress()) {
1330     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1331   }
1332 }
1333 
1334 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1335   for (size_t i = 0; i < num_regions(); i++) {
1336     ShenandoahHeapRegion* current = get_region(i);
1337     blk->heap_region_do(current);
1338   }
1339 }
1340 
1341 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1342 private:
1343   ShenandoahHeap* const _heap;
1344   ShenandoahHeapRegionClosure* const _blk;
1345 
1346   shenandoah_padding(0);
1347   volatile size_t _index;
1348   shenandoah_padding(1);
1349 
1350 public:
1351   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1352           AbstractGangTask("Parallel Region Task"),
1353           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1354 
1355   void work(uint worker_id) {
1356     ShenandoahParallelWorkerSession worker_session(worker_id);
1357     size_t stride = ShenandoahParallelRegionStride;
1358 
1359     size_t max = _heap->num_regions();
1360     while (_index < max) {
1361       size_t cur = Atomic::fetch_and_add(&_index, stride);
1362       size_t start = cur;
1363       size_t end = MIN2(cur + stride, max);
1364       if (start >= max) break;
1365 
1366       for (size_t i = cur; i < end; i++) {
1367         ShenandoahHeapRegion* current = _heap->get_region(i);
1368         _blk->heap_region_do(current);
1369       }
1370     }
1371   }
1372 };
1373 
1374 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1375   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1376   if (num_regions() > ShenandoahParallelRegionStride) {
1377     ShenandoahParallelHeapRegionTask task(blk);
1378     workers()->run_task(&task);
1379   } else {
1380     heap_region_iterate(blk);
1381   }
1382 }
1383 
1384 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1385 private:
1386   ShenandoahMarkingContext* const _ctx;
1387 public:
1388   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1389 
1390   void heap_region_do(ShenandoahHeapRegion* r) {
1391     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1392     if (r->is_active()) {
1393       // Check if region needs updating its TAMS. We have updated it already during concurrent
1394       // reset, so it is very likely we don't need to do another write here.
1395       if (_ctx->top_at_mark_start(r) != r->top()) {
1396         _ctx->capture_top_at_mark_start(r);
1397       }
1398     } else {
1399       assert(_ctx->top_at_mark_start(r) == r->top(),
1400              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1401     }
1402   }
1403 
1404   bool is_thread_safe() { return true; }
1405 };
1406 
1407 void ShenandoahHeap::op_init_mark() {
1408   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1409   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1410 
1411   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1412   assert(!marking_context()->is_complete(), "should not be complete");
1413   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1414 
1415   if (ShenandoahVerify) {
1416     verifier()->verify_before_concmark();
1417   }
1418 
1419   if (VerifyBeforeGC) {
1420     Universe::verify();
1421   }
1422 
1423   set_concurrent_mark_in_progress(true);
1424   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1425   {
1426     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1427     make_parsable(true);
1428   }
1429 
1430   {
1431     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1432     ShenandoahInitMarkUpdateRegionStateClosure cl;
1433     parallel_heap_region_iterate(&cl);
1434   }
1435 
1436   // Make above changes visible to worker threads
1437   OrderAccess::fence();
1438 
1439   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1440 
1441   if (UseTLAB) {
1442     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1443     resize_tlabs();
1444   }
1445 
1446   if (ShenandoahPacing) {
1447     pacer()->setup_for_mark();
1448   }
1449 
1450   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1451   // we need to make sure that all its metadata are marked. alternative is to remark
1452   // thread roots at final mark pause, but it can be potential latency killer.
1453   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1454     ShenandoahCodeRoots::arm_nmethods();
1455   }
1456 }
1457 
1458 void ShenandoahHeap::op_mark() {
1459   concurrent_mark()->mark_from_roots();
1460 }
1461 
1462 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1463 private:
1464   ShenandoahMarkingContext* const _ctx;
1465   ShenandoahHeapLock* const _lock;
1466 
1467 public:
1468   ShenandoahFinalMarkUpdateRegionStateClosure() :
1469     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1470 
1471   void heap_region_do(ShenandoahHeapRegion* r) {
1472     if (r->is_active()) {
1473       // All allocations past TAMS are implicitly live, adjust the region data.
1474       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1475       HeapWord *tams = _ctx->top_at_mark_start(r);
1476       HeapWord *top = r->top();
1477       if (top > tams) {
1478         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1479       }
1480 
1481       // We are about to select the collection set, make sure it knows about
1482       // current pinning status. Also, this allows trashing more regions that
1483       // now have their pinning status dropped.
1484       if (r->is_pinned()) {
1485         if (r->pin_count() == 0) {
1486           ShenandoahHeapLocker locker(_lock);
1487           r->make_unpinned();
1488         }
1489       } else {
1490         if (r->pin_count() > 0) {
1491           ShenandoahHeapLocker locker(_lock);
1492           r->make_pinned();
1493         }
1494       }
1495 
1496       // Remember limit for updating refs. It's guaranteed that we get no
1497       // from-space-refs written from here on.
1498       r->set_update_watermark_at_safepoint(r->top());
1499     } else {
1500       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1501       assert(_ctx->top_at_mark_start(r) == r->top(),
1502              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1503     }
1504   }
1505 
1506   bool is_thread_safe() { return true; }
1507 };
1508 
1509 void ShenandoahHeap::op_final_mark() {
1510   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1511   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1512 
1513   // It is critical that we
1514   // evacuate roots right after finishing marking, so that we don't
1515   // get unmarked objects in the roots.
1516 
1517   if (!cancelled_gc()) {
1518     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1519 
1520     // Marking is completed, deactivate SATB barrier
1521     set_concurrent_mark_in_progress(false);
1522     mark_complete_marking_context();
1523 
1524     parallel_cleaning(false /* full gc*/);
1525 
1526     if (ShenandoahVerify) {
1527       verifier()->verify_roots_no_forwarded();
1528     }
1529 
1530     {
1531       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1532       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1533       parallel_heap_region_iterate(&cl);
1534 
1535       assert_pinned_region_status();
1536     }
1537 
1538     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1539     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1540     // which would be outside the collection set, so no cset writes would happen there.
1541     // Weaker one: new allocations would happen past update watermark, and so less work would
1542     // be needed for reference updates (would update the large filler instead).
1543     {
1544       ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1545       make_parsable(true);
1546     }
1547 
1548     {
1549       ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset);
1550       ShenandoahHeapLocker locker(lock());
1551       _collection_set->clear();
1552       heuristics()->choose_collection_set(_collection_set);
1553     }
1554 
1555     {
1556       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1557       ShenandoahHeapLocker locker(lock());
1558       _free_set->rebuild();
1559     }
1560 
1561     if (!is_degenerated_gc_in_progress()) {
1562       prepare_concurrent_roots();
1563       prepare_concurrent_unloading();
1564     }
1565 
1566     // If collection set has candidates, start evacuation.
1567     // Otherwise, bypass the rest of the cycle.
1568     if (!collection_set()->is_empty()) {
1569       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1570 
1571       if (ShenandoahVerify) {
1572         verifier()->verify_before_evacuation();
1573       }
1574 
1575       set_evacuation_in_progress(true);
1576       // From here on, we need to update references.
1577       set_has_forwarded_objects(true);
1578 
1579       if (!is_degenerated_gc_in_progress()) {
1580         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1581           ShenandoahCodeRoots::arm_nmethods();
1582         }
1583         evacuate_and_update_roots();
1584       }
1585 
1586       if (ShenandoahPacing) {
1587         pacer()->setup_for_evac();
1588       }
1589 
1590       if (ShenandoahVerify) {
1591         // If OOM while evacuating/updating of roots, there is no guarantee of their consistencies
1592         if (!cancelled_gc()) {
1593           ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1594           if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1595             types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1596             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1597             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1598           }
1599 
1600           if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1601             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1602           }
1603           verifier()->verify_roots_no_forwarded_except(types);
1604         }
1605         verifier()->verify_during_evacuation();
1606       }
1607     } else {
1608       if (ShenandoahVerify) {
1609         verifier()->verify_after_concmark();
1610       }
1611 
1612       if (VerifyAfterGC) {
1613         Universe::verify();
1614       }
1615     }
1616 
1617   } else {
1618     // If this cycle was updating references, we need to keep the has_forwarded_objects
1619     // flag on, for subsequent phases to deal with it.
1620     concurrent_mark()->cancel();
1621     set_concurrent_mark_in_progress(false);
1622 
1623     if (process_references()) {
1624       // Abandon reference processing right away: pre-cleaning must have failed.
1625       ReferenceProcessor *rp = ref_processor();
1626       rp->disable_discovery();
1627       rp->abandon_partial_discovery();
1628       rp->verify_no_references_recorded();
1629     }
1630   }
1631 }
1632 
1633 void ShenandoahHeap::op_conc_evac() {
1634   ShenandoahEvacuationTask task(this, _collection_set, true);
1635   workers()->run_task(&task);
1636 }
1637 
1638 void ShenandoahHeap::op_stw_evac() {
1639   ShenandoahEvacuationTask task(this, _collection_set, false);
1640   workers()->run_task(&task);
1641 }
1642 
1643 void ShenandoahHeap::op_updaterefs() {
1644   update_heap_references(true);
1645 }
1646 
1647 void ShenandoahHeap::op_cleanup_early() {
1648   free_set()->recycle_trash();
1649 }
1650 
1651 void ShenandoahHeap::op_cleanup_complete() {
1652   free_set()->recycle_trash();
1653 }
1654 
1655 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1656 private:
1657   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1658   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1659 
1660 public:
1661   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1662     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
1663     _vm_roots(phase),
1664     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()) {}
1665 
1666   void work(uint worker_id) {
1667     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1668     {
1669       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1670       // may race against OopStorage::release() calls.
1671       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1672       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1673     }
1674 
1675     {
1676       ShenandoahEvacuateUpdateRootsClosure<> cl;
1677       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1678       _cld_roots.cld_do(&clds, worker_id);
1679     }
1680   }
1681 };
1682 
1683 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1684 private:
1685   ShenandoahHeap* const _heap;
1686   ShenandoahMarkingContext* const _mark_context;
1687   bool  _evac_in_progress;
1688   Thread* const _thread;
1689   size_t  _dead_counter;
1690 
1691 public:
1692   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1693   void do_oop(oop* p);
1694   void do_oop(narrowOop* p);
1695 
1696   size_t dead_counter() const;
1697   void reset_dead_counter();
1698 };
1699 
1700 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1701   _heap(ShenandoahHeap::heap()),
1702   _mark_context(ShenandoahHeap::heap()->marking_context()),
1703   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1704   _thread(Thread::current()),
1705   _dead_counter(0) {
1706 }
1707 
1708 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1709   const oop obj = RawAccess<>::oop_load(p);
1710   if (!CompressedOops::is_null(obj)) {
1711     if (!_mark_context->is_marked(obj)) {
1712       shenandoah_assert_correct(p, obj);
1713       oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1714       if (obj == old) {
1715         _dead_counter ++;
1716       }
1717     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1718       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1719       if (resolved == obj) {
1720         resolved = _heap->evacuate_object(obj, _thread);
1721       }
1722       Atomic::cmpxchg(p, obj, resolved);
1723       assert(_heap->cancelled_gc() ||
1724              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1725              "Sanity");
1726     }
1727   }
1728 }
1729 
1730 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1731   ShouldNotReachHere();
1732 }
1733 
1734 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1735   return _dead_counter;
1736 }
1737 
1738 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1739   _dead_counter = 0;
1740 }
1741 
1742 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1743 public:
1744   void do_cld(ClassLoaderData* cld) {
1745     cld->is_alive();
1746   }
1747 };
1748 
1749 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1750 public:
1751   void do_nmethod(nmethod* n) {
1752     n->is_unloading();
1753   }
1754 };
1755 
1756 // This task not only evacuates/updates marked weak roots, but also "NULL"
1757 // dead weak roots.
1758 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1759 private:
1760   ShenandoahWeakRoot<true /*concurrent*/>  _jni_roots;
1761   ShenandoahWeakRoot<true /*concurrent*/>  _string_table_roots;
1762   ShenandoahWeakRoot<true /*concurrent*/>  _resolved_method_table_roots;
1763   ShenandoahWeakRoot<true /*concurrent*/>  _vm_roots;
1764 
1765   // Roots related to concurrent class unloading
1766   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1767                                            _cld_roots;
1768   ShenandoahConcurrentNMethodIterator      _nmethod_itr;
1769   ShenandoahConcurrentStringDedupRoots     _dedup_roots;
1770   bool                                     _concurrent_class_unloading;
1771 
1772 public:
1773   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1774     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1775     _jni_roots(OopStorageSet::jni_weak(), phase, ShenandoahPhaseTimings::JNIWeakRoots),
1776     _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots),
1777     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1778     _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots),
1779     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()),
1780     _nmethod_itr(ShenandoahCodeRoots::table()),
1781     _dedup_roots(phase),
1782     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1783     StringTable::reset_dead_counter();
1784     ResolvedMethodTable::reset_dead_counter();
1785     if (_concurrent_class_unloading) {
1786       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1787       _nmethod_itr.nmethods_do_begin();
1788     }
1789   }
1790 
1791   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1792     StringTable::finish_dead_counter();
1793     ResolvedMethodTable::finish_dead_counter();
1794     if (_concurrent_class_unloading) {
1795       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1796       _nmethod_itr.nmethods_do_end();
1797     }
1798   }
1799 
1800   void work(uint worker_id) {
1801     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1802     {
1803       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1804       // may race against OopStorage::release() calls.
1805       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1806       _jni_roots.oops_do(&cl, worker_id);
1807       _vm_roots.oops_do(&cl, worker_id);
1808 
1809       cl.reset_dead_counter();
1810       _string_table_roots.oops_do(&cl, worker_id);
1811       StringTable::inc_dead_counter(cl.dead_counter());
1812 
1813       cl.reset_dead_counter();
1814       _resolved_method_table_roots.oops_do(&cl, worker_id);
1815       ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1816 
1817       // String dedup weak roots
1818       ShenandoahForwardedIsAliveClosure is_alive;
1819       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1820       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1821     }
1822 
1823     // If we are going to perform concurrent class unloading later on, we need to
1824     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1825     // can cleanup immediate garbage sooner.
1826     if (_concurrent_class_unloading) {
1827       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1828       // CLD's holder or evacuate it.
1829       ShenandoahIsCLDAliveClosure is_cld_alive;
1830       _cld_roots.cld_do(&is_cld_alive, worker_id);
1831 
1832       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1833       // The closure calls nmethod->is_unloading(). The is_unloading
1834       // state is cached, therefore, during concurrent class unloading phase,
1835       // we will not touch the metadata of unloading nmethods
1836       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1837       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1838     }
1839   }
1840 };
1841 
1842 void ShenandoahHeap::op_weak_roots() {
1843   if (is_concurrent_weak_root_in_progress()) {
1844     // Concurrent weak root processing
1845     {
1846       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
1847       ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
1848       ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
1849       workers()->run_task(&task);
1850       if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1851         set_concurrent_weak_root_in_progress(false);
1852       }
1853     }
1854 
1855     // Perform handshake to flush out dead oops
1856     {
1857       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1858       ShenandoahRendezvousClosure cl;
1859       Handshake::execute(&cl);
1860     }
1861   }
1862 }
1863 
1864 void ShenandoahHeap::op_class_unloading() {
1865   assert (is_concurrent_weak_root_in_progress() &&
1866           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1867           "Checked by caller");
1868   _unloader.unload();
1869   set_concurrent_weak_root_in_progress(false);
1870 }
1871 
1872 void ShenandoahHeap::op_strong_roots() {
1873   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1874   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1875   workers()->run_task(&task);
1876   set_concurrent_strong_root_in_progress(false);
1877 }
1878 
1879 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1880 private:
1881   ShenandoahMarkingContext* const _ctx;
1882 public:
1883   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1884 
1885   void heap_region_do(ShenandoahHeapRegion* r) {
1886     if (r->is_active()) {
1887       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1888       // anyway to capture any updates that happened since now.
1889       r->clear_live_data();
1890       _ctx->capture_top_at_mark_start(r);
1891     }
1892   }
1893 
1894   bool is_thread_safe() { return true; }
1895 };
1896 
1897 void ShenandoahHeap::op_reset() {
1898   if (ShenandoahPacing) {
1899     pacer()->setup_for_reset();
1900   }
1901   reset_mark_bitmap();
1902 
1903   ShenandoahResetUpdateRegionStateClosure cl;
1904   parallel_heap_region_iterate(&cl);
1905 }
1906 
1907 void ShenandoahHeap::op_preclean() {
1908   if (ShenandoahPacing) {
1909     pacer()->setup_for_preclean();
1910   }
1911   concurrent_mark()->preclean_weak_refs();
1912 }
1913 
1914 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1915   ShenandoahMetricsSnapshot metrics;
1916   metrics.snap_before();
1917 
1918   full_gc()->do_it(cause);
1919   if (UseTLAB) {
1920     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1921     resize_all_tlabs();
1922   }
1923 
1924   metrics.snap_after();
1925 
1926   if (metrics.is_good_progress()) {
1927     _progress_last_gc.set();
1928   } else {
1929     // Nothing to do. Tell the allocation path that we have failed to make
1930     // progress, and it can finally fail.
1931     _progress_last_gc.unset();
1932   }
1933 }
1934 
1935 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1936   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1937   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1938   // some phase, we have to upgrade the Degenerate GC to Full GC.
1939 
1940   clear_cancelled_gc();
1941 
1942   ShenandoahMetricsSnapshot metrics;
1943   metrics.snap_before();
1944 
1945   switch (point) {
1946     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1947     // but enters it at different points, depending on which concurrent phase had
1948     // degenerated.
1949 
1950     case _degenerated_outside_cycle:
1951       // We have degenerated from outside the cycle, which means something is bad with
1952       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1953       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1954       // we can do the most aggressive degen cycle, which includes processing references and
1955       // class unloading, unless those features are explicitly disabled.
1956       //
1957       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1958       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1959       set_process_references(heuristics()->can_process_references());
1960       set_unload_classes(heuristics()->can_unload_classes());
1961 
1962       op_reset();
1963 
1964       op_init_mark();
1965       if (cancelled_gc()) {
1966         op_degenerated_fail();
1967         return;
1968       }
1969 
1970     case _degenerated_mark:
1971       op_final_mark();
1972       if (cancelled_gc()) {
1973         op_degenerated_fail();
1974         return;
1975       }
1976 
1977       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1978         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1979         // be disarmed while conc-roots phase is running.
1980         // TODO: Call op_conc_roots() here instead
1981         ShenandoahCodeRoots::disarm_nmethods();
1982       }
1983 
1984       op_cleanup_early();
1985 
1986     case _degenerated_evac:
1987       // If heuristics thinks we should do the cycle, this flag would be set,
1988       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1989       if (is_evacuation_in_progress()) {
1990 
1991         // Degeneration under oom-evac protocol might have left some objects in
1992         // collection set un-evacuated. Restart evacuation from the beginning to
1993         // capture all objects. For all the objects that are already evacuated,
1994         // it would be a simple check, which is supposed to be fast. This is also
1995         // safe to do even without degeneration, as CSet iterator is at beginning
1996         // in preparation for evacuation anyway.
1997         //
1998         // Before doing that, we need to make sure we never had any cset-pinned
1999         // regions. This may happen if allocation failure happened when evacuating
2000         // the about-to-be-pinned object, oom-evac protocol left the object in
2001         // the collection set, and then the pin reached the cset region. If we continue
2002         // the cycle here, we would trash the cset and alive objects in it. To avoid
2003         // it, we fail degeneration right away and slide into Full GC to recover.
2004 
2005         {
2006           sync_pinned_region_status();
2007           collection_set()->clear_current_index();
2008 
2009           ShenandoahHeapRegion* r;
2010           while ((r = collection_set()->next()) != NULL) {
2011             if (r->is_pinned()) {
2012               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2013               op_degenerated_fail();
2014               return;
2015             }
2016           }
2017 
2018           collection_set()->clear_current_index();
2019         }
2020 
2021         op_stw_evac();
2022         if (cancelled_gc()) {
2023           op_degenerated_fail();
2024           return;
2025         }
2026       }
2027 
2028       // If heuristics thinks we should do the cycle, this flag would be set,
2029       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2030       if (has_forwarded_objects()) {
2031         op_init_updaterefs();
2032         if (cancelled_gc()) {
2033           op_degenerated_fail();
2034           return;
2035         }
2036       }
2037 
2038     case _degenerated_updaterefs:
2039       if (has_forwarded_objects()) {
2040         op_final_updaterefs();
2041         if (cancelled_gc()) {
2042           op_degenerated_fail();
2043           return;
2044         }
2045       }
2046 
2047       op_cleanup_complete();
2048       break;
2049 
2050     default:
2051       ShouldNotReachHere();
2052   }
2053 
2054   if (ShenandoahVerify) {
2055     verifier()->verify_after_degenerated();
2056   }
2057 
2058   if (VerifyAfterGC) {
2059     Universe::verify();
2060   }
2061 
2062   metrics.snap_after();
2063 
2064   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2065   // because that probably means the heap is overloaded and/or fragmented.
2066   if (!metrics.is_good_progress()) {
2067     _progress_last_gc.unset();
2068     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2069     op_degenerated_futile();
2070   } else {
2071     _progress_last_gc.set();
2072   }
2073 }
2074 
2075 void ShenandoahHeap::op_degenerated_fail() {
2076   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2077   shenandoah_policy()->record_degenerated_upgrade_to_full();
2078   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2079 }
2080 
2081 void ShenandoahHeap::op_degenerated_futile() {
2082   shenandoah_policy()->record_degenerated_upgrade_to_full();
2083   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2084 }
2085 
2086 void ShenandoahHeap::force_satb_flush_all_threads() {
2087   if (!is_concurrent_mark_in_progress()) {
2088     // No need to flush SATBs
2089     return;
2090   }
2091 
2092   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2093     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2094   }
2095   // The threads are not "acquiring" their thread-local data, but it does not
2096   // hurt to "release" the updates here anyway.
2097   OrderAccess::fence();
2098 }
2099 
2100 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2101   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2102     ShenandoahThreadLocalData::set_gc_state(t, state);
2103   }
2104 }
2105 
2106 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2107   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2108   _gc_state.set_cond(mask, value);
2109   set_gc_state_all_threads(_gc_state.raw_value());
2110 }
2111 
2112 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2113   if (has_forwarded_objects()) {
2114     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2115   } else {
2116     set_gc_state_mask(MARKING, in_progress);
2117   }
2118   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2119 }
2120 
2121 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2122   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2123   set_gc_state_mask(EVACUATION, in_progress);
2124 }
2125 
2126 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2127   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2128   if (in_progress) {
2129     _concurrent_strong_root_in_progress.set();
2130   } else {
2131     _concurrent_strong_root_in_progress.unset();
2132   }
2133 }
2134 
2135 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2136   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2137   if (in_progress) {
2138     _concurrent_weak_root_in_progress.set();
2139   } else {
2140     _concurrent_weak_root_in_progress.unset();
2141   }
2142 }
2143 
2144 void ShenandoahHeap::ref_processing_init() {
2145   assert(_max_workers > 0, "Sanity");
2146 
2147   _ref_processor =
2148     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2149                            _ref_proc_mt_processing, // MT processing
2150                            _max_workers,            // Degree of MT processing
2151                            _ref_proc_mt_discovery,  // MT discovery
2152                            _max_workers,            // Degree of MT discovery
2153                            false,                   // Reference discovery is not atomic
2154                            NULL,                    // No closure, should be installed before use
2155                            true);                   // Scale worker threads
2156 
2157   shenandoah_assert_rp_isalive_not_installed();
2158 }
2159 
2160 GCTracer* ShenandoahHeap::tracer() {
2161   return shenandoah_policy()->tracer();
2162 }
2163 
2164 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2165   return _free_set->used();
2166 }
2167 
2168 bool ShenandoahHeap::try_cancel_gc() {
2169   while (true) {
2170     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2171     if (prev == CANCELLABLE) return true;
2172     else if (prev == CANCELLED) return false;
2173     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2174     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2175     if (Thread::current()->is_Java_thread()) {
2176       // We need to provide a safepoint here, otherwise we might
2177       // spin forever if a SP is pending.
2178       ThreadBlockInVM sp(JavaThread::current());
2179       SpinPause();
2180     }
2181   }
2182 }
2183 
2184 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2185   if (try_cancel_gc()) {
2186     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2187     log_info(gc)("%s", msg.buffer());
2188     Events::log(Thread::current(), "%s", msg.buffer());
2189   }
2190 }
2191 
2192 uint ShenandoahHeap::max_workers() {
2193   return _max_workers;
2194 }
2195 
2196 void ShenandoahHeap::stop() {
2197   // The shutdown sequence should be able to terminate when GC is running.
2198 
2199   // Step 0. Notify policy to disable event recording.
2200   _shenandoah_policy->record_shutdown();
2201 
2202   // Step 1. Notify control thread that we are in shutdown.
2203   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2204   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2205   control_thread()->prepare_for_graceful_shutdown();
2206 
2207   // Step 2. Notify GC workers that we are cancelling GC.
2208   cancel_gc(GCCause::_shenandoah_stop_vm);
2209 
2210   // Step 3. Wait until GC worker exits normally.
2211   control_thread()->stop();
2212 
2213   // Step 4. Stop String Dedup thread if it is active
2214   if (ShenandoahStringDedup::is_enabled()) {
2215     ShenandoahStringDedup::stop();
2216   }
2217 }
2218 
2219 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2220   if (!unload_classes()) return;
2221 
2222   // Unload classes and purge SystemDictionary.
2223   {
2224     ShenandoahGCPhase phase(full_gc ?
2225                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
2226                             ShenandoahPhaseTimings::purge_class_unload);
2227     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2228 
2229     ShenandoahIsAliveSelector is_alive;
2230     uint num_workers = _workers->active_workers();
2231     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2232     _workers->run_task(&unlink_task);
2233   }
2234 
2235   {
2236     ShenandoahGCPhase phase(full_gc ?
2237                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2238                             ShenandoahPhaseTimings::purge_cldg);
2239     ClassLoaderDataGraph::purge();
2240   }
2241   // Resize and verify metaspace
2242   MetaspaceGC::compute_new_size();
2243   MetaspaceUtils::verify_metrics();
2244 }
2245 
2246 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2247 // so they should not have forwarded oops.
2248 // However, we do need to "null" dead oops in the roots, if can not be done
2249 // in concurrent cycles.
2250 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2251   ShenandoahGCPhase root_phase(full_gc ?
2252                                ShenandoahPhaseTimings::full_gc_purge :
2253                                ShenandoahPhaseTimings::purge);
2254   uint num_workers = _workers->active_workers();
2255   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2256                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2257                                                ShenandoahPhaseTimings::purge_weak_par;
2258   ShenandoahGCPhase phase(timing_phase);
2259   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2260 
2261   // Cleanup weak roots
2262   if (has_forwarded_objects()) {
2263     ShenandoahForwardedIsAliveClosure is_alive;
2264     ShenandoahUpdateRefsClosure keep_alive;
2265     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2266       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2267     _workers->run_task(&cleaning_task);
2268   } else {
2269     ShenandoahIsAliveClosure is_alive;
2270 #ifdef ASSERT
2271     ShenandoahAssertNotForwardedClosure verify_cl;
2272     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2273       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2274 #else
2275     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2276       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2277 #endif
2278     _workers->run_task(&cleaning_task);
2279   }
2280 }
2281 
2282 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2283   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2284   stw_process_weak_roots(full_gc);
2285   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2286     stw_unload_classes(full_gc);
2287   }
2288 }
2289 
2290 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2291   set_gc_state_mask(HAS_FORWARDED, cond);
2292 }
2293 
2294 void ShenandoahHeap::set_process_references(bool pr) {
2295   _process_references.set_cond(pr);
2296 }
2297 
2298 void ShenandoahHeap::set_unload_classes(bool uc) {
2299   _unload_classes.set_cond(uc);
2300 }
2301 
2302 bool ShenandoahHeap::process_references() const {
2303   return _process_references.is_set();
2304 }
2305 
2306 bool ShenandoahHeap::unload_classes() const {
2307   return _unload_classes.is_set();
2308 }
2309 
2310 address ShenandoahHeap::in_cset_fast_test_addr() {
2311   ShenandoahHeap* heap = ShenandoahHeap::heap();
2312   assert(heap->collection_set() != NULL, "Sanity");
2313   return (address) heap->collection_set()->biased_map_address();
2314 }
2315 
2316 address ShenandoahHeap::cancelled_gc_addr() {
2317   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2318 }
2319 
2320 address ShenandoahHeap::gc_state_addr() {
2321   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2322 }
2323 
2324 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2325   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2326 }
2327 
2328 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2329   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2330 }
2331 
2332 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2333   _degenerated_gc_in_progress.set_cond(in_progress);
2334 }
2335 
2336 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2337   _full_gc_in_progress.set_cond(in_progress);
2338 }
2339 
2340 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2341   assert (is_full_gc_in_progress(), "should be");
2342   _full_gc_move_in_progress.set_cond(in_progress);
2343 }
2344 
2345 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2346   set_gc_state_mask(UPDATEREFS, in_progress);
2347 }
2348 
2349 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2350   ShenandoahCodeRoots::register_nmethod(nm);
2351 }
2352 
2353 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2354   ShenandoahCodeRoots::unregister_nmethod(nm);
2355 }
2356 
2357 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2358   ShenandoahCodeRoots::flush_nmethod(nm);
2359 }
2360 
2361 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2362   heap_region_containing(o)->record_pin();
2363   return o;
2364 }
2365 
2366 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2367   heap_region_containing(o)->record_unpin();
2368 }
2369 
2370 void ShenandoahHeap::sync_pinned_region_status() {
2371   ShenandoahHeapLocker locker(lock());
2372 
2373   for (size_t i = 0; i < num_regions(); i++) {
2374     ShenandoahHeapRegion *r = get_region(i);
2375     if (r->is_active()) {
2376       if (r->is_pinned()) {
2377         if (r->pin_count() == 0) {
2378           r->make_unpinned();
2379         }
2380       } else {
2381         if (r->pin_count() > 0) {
2382           r->make_pinned();
2383         }
2384       }
2385     }
2386   }
2387 
2388   assert_pinned_region_status();
2389 }
2390 
2391 #ifdef ASSERT
2392 void ShenandoahHeap::assert_pinned_region_status() {
2393   for (size_t i = 0; i < num_regions(); i++) {
2394     ShenandoahHeapRegion* r = get_region(i);
2395     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2396            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2397   }
2398 }
2399 #endif
2400 
2401 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2402   return _gc_timer;
2403 }
2404 
2405 void ShenandoahHeap::prepare_concurrent_roots() {
2406   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2407   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2408     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2409     set_concurrent_weak_root_in_progress(true);
2410   }
2411 }
2412 
2413 void ShenandoahHeap::prepare_concurrent_unloading() {
2414   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2415   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2416     _unloader.prepare();
2417   }
2418 }
2419 
2420 void ShenandoahHeap::finish_concurrent_unloading() {
2421   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2422   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2423     _unloader.finish();
2424   }
2425 }
2426 
2427 #ifdef ASSERT
2428 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2429   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2430 
2431   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2432     if (UseDynamicNumberOfGCThreads) {
2433       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2434     } else {
2435       // Use ParallelGCThreads inside safepoints
2436       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2437     }
2438   } else {
2439     if (UseDynamicNumberOfGCThreads) {
2440       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2441     } else {
2442       // Use ConcGCThreads outside safepoints
2443       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2444     }
2445   }
2446 }
2447 #endif
2448 
2449 ShenandoahVerifier* ShenandoahHeap::verifier() {
2450   guarantee(ShenandoahVerify, "Should be enabled");
2451   assert (_verifier != NULL, "sanity");
2452   return _verifier;
2453 }
2454 
2455 template<class T>
2456 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2457 private:
2458   T cl;
2459   ShenandoahHeap* _heap;
2460   ShenandoahRegionIterator* _regions;
2461   bool _concurrent;
2462 public:
2463   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2464     AbstractGangTask("Concurrent Update References Task"),
2465     cl(T()),
2466     _heap(ShenandoahHeap::heap()),
2467     _regions(regions),
2468     _concurrent(concurrent) {
2469   }
2470 
2471   void work(uint worker_id) {
2472     if (_concurrent) {
2473       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2474       SuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2475       do_work();
2476     } else {
2477       ShenandoahParallelWorkerSession worker_session(worker_id);
2478       do_work();
2479     }
2480   }
2481 
2482 private:
2483   void do_work() {
2484     ShenandoahHeapRegion* r = _regions->next();
2485     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2486     while (r != NULL) {
2487       HeapWord* update_watermark = r->get_update_watermark();
2488       assert (update_watermark >= r->bottom(), "sanity");
2489       if (r->is_active() && !r->is_cset()) {
2490         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2491       }
2492       if (ShenandoahPacing) {
2493         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2494       }
2495       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2496         return;
2497       }
2498       r = _regions->next();
2499     }
2500   }
2501 };
2502 
2503 void ShenandoahHeap::update_heap_references(bool concurrent) {
2504   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2505   workers()->run_task(&task);
2506 }
2507 
2508 void ShenandoahHeap::op_init_updaterefs() {
2509   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2510 
2511   set_evacuation_in_progress(false);
2512 
2513   {
2514     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2515     retire_and_reset_gclabs();
2516   }
2517 
2518   if (ShenandoahVerify) {
2519     if (!is_degenerated_gc_in_progress()) {
2520       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2521     }
2522     verifier()->verify_before_updaterefs();
2523   }
2524 
2525   set_update_refs_in_progress(true);
2526 
2527   _update_refs_iterator.reset();
2528 
2529   if (ShenandoahPacing) {
2530     pacer()->setup_for_updaterefs();
2531   }
2532 }
2533 
2534 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2535 private:
2536   ShenandoahHeapLock* const _lock;
2537 
2538 public:
2539   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2540 
2541   void heap_region_do(ShenandoahHeapRegion* r) {
2542     // Drop unnecessary "pinned" state from regions that does not have CP marks
2543     // anymore, as this would allow trashing them.
2544 
2545     if (r->is_active()) {
2546       if (r->is_pinned()) {
2547         if (r->pin_count() == 0) {
2548           ShenandoahHeapLocker locker(_lock);
2549           r->make_unpinned();
2550         }
2551       } else {
2552         if (r->pin_count() > 0) {
2553           ShenandoahHeapLocker locker(_lock);
2554           r->make_pinned();
2555         }
2556       }
2557     }
2558   }
2559 
2560   bool is_thread_safe() { return true; }
2561 };
2562 
2563 void ShenandoahHeap::op_final_updaterefs() {
2564   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2565 
2566   finish_concurrent_unloading();
2567 
2568   // Check if there is left-over work, and finish it
2569   if (_update_refs_iterator.has_next()) {
2570     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2571 
2572     // Finish updating references where we left off.
2573     clear_cancelled_gc();
2574     update_heap_references(false);
2575   }
2576 
2577   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2578   // everything. On degenerated paths, cancelled gc would not be set anyway.
2579   if (cancelled_gc()) {
2580     clear_cancelled_gc();
2581   }
2582   assert(!cancelled_gc(), "Should have been done right before");
2583 
2584   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2585     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2586   }
2587 
2588   if (is_degenerated_gc_in_progress()) {
2589     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2590   } else {
2591     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2592   }
2593 
2594   // Has to be done before cset is clear
2595   if (ShenandoahVerify) {
2596     verifier()->verify_roots_in_to_space();
2597   }
2598 
2599   {
2600     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2601     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2602     parallel_heap_region_iterate(&cl);
2603 
2604     assert_pinned_region_status();
2605   }
2606 
2607   {
2608     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2609     trash_cset_regions();
2610   }
2611 
2612   set_has_forwarded_objects(false);
2613   set_update_refs_in_progress(false);
2614 
2615   if (ShenandoahVerify) {
2616     verifier()->verify_after_updaterefs();
2617   }
2618 
2619   if (VerifyAfterGC) {
2620     Universe::verify();
2621   }
2622 
2623   {
2624     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2625     ShenandoahHeapLocker locker(lock());
2626     _free_set->rebuild();
2627   }
2628 }
2629 
2630 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2631   print_on(st);
2632   print_heap_regions_on(st);
2633 }
2634 
2635 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2636   size_t slice = r->index() / _bitmap_regions_per_slice;
2637 
2638   size_t regions_from = _bitmap_regions_per_slice * slice;
2639   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2640   for (size_t g = regions_from; g < regions_to; g++) {
2641     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2642     if (skip_self && g == r->index()) continue;
2643     if (get_region(g)->is_committed()) {
2644       return true;
2645     }
2646   }
2647   return false;
2648 }
2649 
2650 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2651   shenandoah_assert_heaplocked();
2652 
2653   // Bitmaps in special regions do not need commits
2654   if (_bitmap_region_special) {
2655     return true;
2656   }
2657 
2658   if (is_bitmap_slice_committed(r, true)) {
2659     // Some other region from the group is already committed, meaning the bitmap
2660     // slice is already committed, we exit right away.
2661     return true;
2662   }
2663 
2664   // Commit the bitmap slice:
2665   size_t slice = r->index() / _bitmap_regions_per_slice;
2666   size_t off = _bitmap_bytes_per_slice * slice;
2667   size_t len = _bitmap_bytes_per_slice;
2668   char* start = (char*) _bitmap_region.start() + off;
2669 
2670   if (!os::commit_memory(start, len, false)) {
2671     return false;
2672   }
2673 
2674   if (AlwaysPreTouch) {
2675     os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2676   }
2677 
2678   return true;
2679 }
2680 
2681 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2682   shenandoah_assert_heaplocked();
2683 
2684   // Bitmaps in special regions do not need uncommits
2685   if (_bitmap_region_special) {
2686     return true;
2687   }
2688 
2689   if (is_bitmap_slice_committed(r, true)) {
2690     // Some other region from the group is still committed, meaning the bitmap
2691     // slice is should stay committed, exit right away.
2692     return true;
2693   }
2694 
2695   // Uncommit the bitmap slice:
2696   size_t slice = r->index() / _bitmap_regions_per_slice;
2697   size_t off = _bitmap_bytes_per_slice * slice;
2698   size_t len = _bitmap_bytes_per_slice;
2699   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2700     return false;
2701   }
2702   return true;
2703 }
2704 
2705 void ShenandoahHeap::safepoint_synchronize_begin() {
2706   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2707     SuspendibleThreadSet::synchronize();
2708   }
2709 }
2710 
2711 void ShenandoahHeap::safepoint_synchronize_end() {
2712   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2713     SuspendibleThreadSet::desynchronize();
2714   }
2715 }
2716 
2717 void ShenandoahHeap::vmop_entry_init_mark() {
2718   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2719   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
2720 
2721   try_inject_alloc_failure();
2722   VM_ShenandoahInitMark op;
2723   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2724 }
2725 
2726 void ShenandoahHeap::vmop_entry_final_mark() {
2727   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2728   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
2729 
2730   try_inject_alloc_failure();
2731   VM_ShenandoahFinalMarkStartEvac op;
2732   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2733 }
2734 
2735 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2736   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2737   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
2738 
2739   try_inject_alloc_failure();
2740   VM_ShenandoahInitUpdateRefs op;
2741   VMThread::execute(&op);
2742 }
2743 
2744 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2745   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2746   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
2747 
2748   try_inject_alloc_failure();
2749   VM_ShenandoahFinalUpdateRefs op;
2750   VMThread::execute(&op);
2751 }
2752 
2753 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2754   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2755   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
2756 
2757   try_inject_alloc_failure();
2758   VM_ShenandoahFullGC op(cause);
2759   VMThread::execute(&op);
2760 }
2761 
2762 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2763   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2764   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
2765 
2766   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2767   VMThread::execute(&degenerated_gc);
2768 }
2769 
2770 void ShenandoahHeap::entry_init_mark() {
2771   const char* msg = init_mark_event_message();
2772   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
2773   EventMark em("%s", msg);
2774 
2775   ShenandoahWorkerScope scope(workers(),
2776                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2777                               "init marking");
2778 
2779   op_init_mark();
2780 }
2781 
2782 void ShenandoahHeap::entry_final_mark() {
2783   const char* msg = final_mark_event_message();
2784   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
2785   EventMark em("%s", msg);
2786 
2787   ShenandoahWorkerScope scope(workers(),
2788                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2789                               "final marking");
2790 
2791   op_final_mark();
2792 }
2793 
2794 void ShenandoahHeap::entry_init_updaterefs() {
2795   static const char* msg = "Pause Init Update Refs";
2796   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
2797   EventMark em("%s", msg);
2798 
2799   // No workers used in this phase, no setup required
2800 
2801   op_init_updaterefs();
2802 }
2803 
2804 void ShenandoahHeap::entry_final_updaterefs() {
2805   static const char* msg = "Pause Final Update Refs";
2806   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
2807   EventMark em("%s", msg);
2808 
2809   ShenandoahWorkerScope scope(workers(),
2810                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2811                               "final reference update");
2812 
2813   op_final_updaterefs();
2814 }
2815 
2816 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2817   static const char* msg = "Pause Full";
2818   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
2819   EventMark em("%s", msg);
2820 
2821   ShenandoahWorkerScope scope(workers(),
2822                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2823                               "full gc");
2824 
2825   op_full(cause);
2826 }
2827 
2828 void ShenandoahHeap::entry_degenerated(int point) {
2829   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2830   const char* msg = degen_event_message(dpoint);
2831   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
2832   EventMark em("%s", msg);
2833 
2834   ShenandoahWorkerScope scope(workers(),
2835                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2836                               "stw degenerated gc");
2837 
2838   set_degenerated_gc_in_progress(true);
2839   op_degenerated(dpoint);
2840   set_degenerated_gc_in_progress(false);
2841 }
2842 
2843 void ShenandoahHeap::entry_mark() {
2844   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2845 
2846   const char* msg = conc_mark_event_message();
2847   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
2848   EventMark em("%s", msg);
2849 
2850   ShenandoahWorkerScope scope(workers(),
2851                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2852                               "concurrent marking");
2853 
2854   try_inject_alloc_failure();
2855   op_mark();
2856 }
2857 
2858 void ShenandoahHeap::entry_evac() {
2859   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2860 
2861   static const char* msg = "Concurrent evacuation";
2862   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
2863   EventMark em("%s", msg);
2864 
2865   ShenandoahWorkerScope scope(workers(),
2866                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2867                               "concurrent evacuation");
2868 
2869   try_inject_alloc_failure();
2870   op_conc_evac();
2871 }
2872 
2873 void ShenandoahHeap::entry_updaterefs() {
2874   static const char* msg = "Concurrent update references";
2875   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
2876   EventMark em("%s", msg);
2877 
2878   ShenandoahWorkerScope scope(workers(),
2879                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2880                               "concurrent reference update");
2881 
2882   try_inject_alloc_failure();
2883   op_updaterefs();
2884 }
2885 
2886 void ShenandoahHeap::entry_weak_roots() {
2887   static const char* msg = "Concurrent weak roots";
2888   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
2889   EventMark em("%s", msg);
2890 
2891   ShenandoahWorkerScope scope(workers(),
2892                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2893                               "concurrent weak root");
2894 
2895   try_inject_alloc_failure();
2896   op_weak_roots();
2897 }
2898 
2899 void ShenandoahHeap::entry_class_unloading() {
2900   static const char* msg = "Concurrent class unloading";
2901   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
2902   EventMark em("%s", msg);
2903 
2904   ShenandoahWorkerScope scope(workers(),
2905                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2906                               "concurrent class unloading");
2907 
2908   try_inject_alloc_failure();
2909   op_class_unloading();
2910 }
2911 
2912 void ShenandoahHeap::entry_strong_roots() {
2913   static const char* msg = "Concurrent strong roots";
2914   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
2915   EventMark em("%s", msg);
2916 
2917   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
2918 
2919   ShenandoahWorkerScope scope(workers(),
2920                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2921                               "concurrent strong root");
2922 
2923   try_inject_alloc_failure();
2924   op_strong_roots();
2925 }
2926 
2927 void ShenandoahHeap::entry_cleanup_early() {
2928   static const char* msg = "Concurrent cleanup";
2929   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
2930   EventMark em("%s", msg);
2931 
2932   // This phase does not use workers, no need for setup
2933 
2934   try_inject_alloc_failure();
2935   op_cleanup_early();
2936 }
2937 
2938 void ShenandoahHeap::entry_cleanup_complete() {
2939   static const char* msg = "Concurrent cleanup";
2940   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
2941   EventMark em("%s", msg);
2942 
2943   // This phase does not use workers, no need for setup
2944 
2945   try_inject_alloc_failure();
2946   op_cleanup_complete();
2947 }
2948 
2949 void ShenandoahHeap::entry_reset() {
2950   static const char* msg = "Concurrent reset";
2951   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
2952   EventMark em("%s", msg);
2953 
2954   ShenandoahWorkerScope scope(workers(),
2955                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2956                               "concurrent reset");
2957 
2958   try_inject_alloc_failure();
2959   op_reset();
2960 }
2961 
2962 void ShenandoahHeap::entry_preclean() {
2963   if (ShenandoahPreclean && process_references()) {
2964     static const char* msg = "Concurrent precleaning";
2965     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_preclean);
2966     EventMark em("%s", msg);
2967 
2968     ShenandoahWorkerScope scope(workers(),
2969                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2970                                 "concurrent preclean",
2971                                 /* check_workers = */ false);
2972 
2973     try_inject_alloc_failure();
2974     op_preclean();
2975   }
2976 }
2977 
2978 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2979   static const char *msg = "Concurrent uncommit";
2980   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2981   EventMark em("%s", msg);
2982 
2983   op_uncommit(shrink_before);
2984 }
2985 
2986 void ShenandoahHeap::try_inject_alloc_failure() {
2987   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2988     _inject_alloc_failure.set();
2989     os::naked_short_sleep(1);
2990     if (cancelled_gc()) {
2991       log_info(gc)("Allocation failure was successfully injected");
2992     }
2993   }
2994 }
2995 
2996 bool ShenandoahHeap::should_inject_alloc_failure() {
2997   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2998 }
2999 
3000 void ShenandoahHeap::initialize_serviceability() {
3001   _memory_pool = new ShenandoahMemoryPool(this);
3002   _cycle_memory_manager.add_pool(_memory_pool);
3003   _stw_memory_manager.add_pool(_memory_pool);
3004 }
3005 
3006 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
3007   GrowableArray<GCMemoryManager*> memory_managers(2);
3008   memory_managers.append(&_cycle_memory_manager);
3009   memory_managers.append(&_stw_memory_manager);
3010   return memory_managers;
3011 }
3012 
3013 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3014   GrowableArray<MemoryPool*> memory_pools(1);
3015   memory_pools.append(_memory_pool);
3016   return memory_pools;
3017 }
3018 
3019 MemoryUsage ShenandoahHeap::memory_usage() {
3020   return _memory_pool->get_memory_usage();
3021 }
3022 
3023 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3024   _heap(ShenandoahHeap::heap()),
3025   _index(0) {}
3026 
3027 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3028   _heap(heap),
3029   _index(0) {}
3030 
3031 void ShenandoahRegionIterator::reset() {
3032   _index = 0;
3033 }
3034 
3035 bool ShenandoahRegionIterator::has_next() const {
3036   return _index < _heap->num_regions();
3037 }
3038 
3039 char ShenandoahHeap::gc_state() const {
3040   return _gc_state.raw_value();
3041 }
3042 
3043 void ShenandoahHeap::deduplicate_string(oop str) {
3044   assert(java_lang_String::is_instance(str), "invariant");
3045 
3046   if (ShenandoahStringDedup::is_enabled()) {
3047     ShenandoahStringDedup::deduplicate(str);
3048   }
3049 }
3050 
3051 const char* ShenandoahHeap::init_mark_event_message() const {
3052   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3053 
3054   bool proc_refs = process_references();
3055   bool unload_cls = unload_classes();
3056 
3057   if (proc_refs && unload_cls) {
3058     return "Pause Init Mark (process weakrefs) (unload classes)";
3059   } else if (proc_refs) {
3060     return "Pause Init Mark (process weakrefs)";
3061   } else if (unload_cls) {
3062     return "Pause Init Mark (unload classes)";
3063   } else {
3064     return "Pause Init Mark";
3065   }
3066 }
3067 
3068 const char* ShenandoahHeap::final_mark_event_message() const {
3069   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3070 
3071   bool proc_refs = process_references();
3072   bool unload_cls = unload_classes();
3073 
3074   if (proc_refs && unload_cls) {
3075     return "Pause Final Mark (process weakrefs) (unload classes)";
3076   } else if (proc_refs) {
3077     return "Pause Final Mark (process weakrefs)";
3078   } else if (unload_cls) {
3079     return "Pause Final Mark (unload classes)";
3080   } else {
3081     return "Pause Final Mark";
3082   }
3083 }
3084 
3085 const char* ShenandoahHeap::conc_mark_event_message() const {
3086   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3087 
3088   bool proc_refs = process_references();
3089   bool unload_cls = unload_classes();
3090 
3091   if (proc_refs && unload_cls) {
3092     return "Concurrent marking (process weakrefs) (unload classes)";
3093   } else if (proc_refs) {
3094     return "Concurrent marking (process weakrefs)";
3095   } else if (unload_cls) {
3096     return "Concurrent marking (unload classes)";
3097   } else {
3098     return "Concurrent marking";
3099   }
3100 }
3101 
3102 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3103   switch (point) {
3104     case _degenerated_unset:
3105       return "Pause Degenerated GC (<UNSET>)";
3106     case _degenerated_outside_cycle:
3107       return "Pause Degenerated GC (Outside of Cycle)";
3108     case _degenerated_mark:
3109       return "Pause Degenerated GC (Mark)";
3110     case _degenerated_evac:
3111       return "Pause Degenerated GC (Evacuation)";
3112     case _degenerated_updaterefs:
3113       return "Pause Degenerated GC (Update Refs)";
3114     default:
3115       ShouldNotReachHere();
3116       return "ERROR";
3117   }
3118 }
3119 
3120 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3121 #ifdef ASSERT
3122   assert(_liveness_cache != NULL, "sanity");
3123   assert(worker_id < _max_workers, "sanity");
3124   for (uint i = 0; i < num_regions(); i++) {
3125     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3126   }
3127 #endif
3128   return _liveness_cache[worker_id];
3129 }
3130 
3131 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3132   assert(worker_id < _max_workers, "sanity");
3133   assert(_liveness_cache != NULL, "sanity");
3134   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3135   for (uint i = 0; i < num_regions(); i++) {
3136     ShenandoahLiveData live = ld[i];
3137     if (live > 0) {
3138       ShenandoahHeapRegion* r = get_region(i);
3139       r->increase_live_data_gc_words(live);
3140       ld[i] = 0;
3141     }
3142   }
3143 }