< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page
@@ -33,46 +33,56 @@
  #include "gc/shared/memAllocator.hpp"
  #include "gc/shared/plab.hpp"
  #include "gc/shared/tlab_globals.hpp"
  
  #include "gc/shenandoah/shenandoahBarrierSet.hpp"
+ #include "gc/shenandoah/shenandoahCardTable.hpp"
  #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  #include "gc/shenandoah/shenandoahControlThread.hpp"
+ #include "gc/shenandoah/shenandoahRegulatorThread.hpp"
  #include "gc/shenandoah/shenandoahFreeSet.hpp"
+ #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  #include "gc/shenandoah/shenandoahInitLogger.hpp"
  #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  #include "gc/shenandoah/shenandoahMetrics.hpp"
  #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+ #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  #include "gc/shenandoah/shenandoahPadding.hpp"
  #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
+ #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  #include "gc/shenandoah/shenandoahStringDedup.hpp"
  #include "gc/shenandoah/shenandoahSTWMark.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
  #include "gc/shenandoah/shenandoahVerifier.hpp"
  #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  #include "gc/shenandoah/shenandoahVMOperations.hpp"
  #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
+ #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+ #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
  #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
+ 
  #if INCLUDE_JFR
  #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  #endif
  
+ #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
+ 
  #include "classfile/systemDictionary.hpp"
  #include "memory/classLoaderMetaspace.hpp"
  #include "memory/metaspaceUtils.hpp"
  #include "oops/compressedOops.inline.hpp"
  #include "prims/jvmtiTagMap.hpp"

@@ -155,13 +165,10 @@
    _num_regions = ShenandoahHeapRegion::region_count();
    assert(_num_regions == (max_byte_size / reg_size_bytes),
           "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
           _num_regions, max_byte_size, reg_size_bytes);
  
-   // Now we know the number of regions, initialize the heuristics.
-   initialize_heuristics();
- 
    size_t num_committed_regions = init_byte_size / reg_size_bytes;
    num_committed_regions = MIN2(num_committed_regions, _num_regions);
    assert(num_committed_regions <= _num_regions, "sanity");
    _initial_size = num_committed_regions * reg_size_bytes;
  

@@ -173,10 +180,14 @@
    // Default to max heap size.
    _soft_max_size = _num_regions * reg_size_bytes;
  
    _committed = _initial_size;
  
+   // Now we know the number of regions and heap sizes, initialize the heuristics.
+   initialize_generations();
+   initialize_heuristics();
+ 
    size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
    size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
    size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
  
    //

@@ -207,10 +218,40 @@
    if (!_heap_region_special) {
      os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
                                "Cannot commit heap memory");
    }
  
+   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
+ 
+   //
+   // After reserving the Java heap, create the card table, barriers, and workers, in dependency order
+   //
+   if (mode()->is_generational()) {
+     ShenandoahDirectCardMarkRememberedSet *rs;
+     ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table();
+     size_t card_count = card_table->cards_required(heap_rs.size() / HeapWordSize) - 1;
+     rs = new ShenandoahDirectCardMarkRememberedSet(ShenandoahBarrierSet::barrier_set()->card_table(), card_count);
+     _card_scan = new ShenandoahScanRemembered<ShenandoahDirectCardMarkRememberedSet>(rs);
+   }
+ 
+   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
+                             /* are_GC_task_threads */ true,
+                             /* are_ConcurrentGC_threads */ true);
+   if (_workers == NULL) {
+     vm_exit_during_initialization("Failed necessary allocation.");
+   } else {
+     _workers->initialize_workers();
+   }
+ 
+   if (ParallelGCThreads > 1) {
+     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
+                                                 ParallelGCThreads,
+                       /* are_GC_task_threads */ false,
+                  /* are_ConcurrentGC_threads */ false);
+     _safepoint_workers->initialize_workers();
+   }
+ 
    //
    // Reserve and commit memory for bitmap(s)
    //
  
    _bitmap_size = ShenandoahMarkBitMap::compute_size(heap_rs.size());

@@ -250,11 +291,11 @@
    if (!_bitmap_region_special) {
      os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
                                "Cannot commit bitmap memory");
    }
  
-   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
+   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
  
    if (ShenandoahVerify) {
      ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
      if (!verify_bitmap.special()) {
        os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,

@@ -396,24 +437,38 @@
    } else {
      _pacer = NULL;
    }
  
    _control_thread = new ShenandoahControlThread();
+   _regulator_thread = new ShenandoahRegulatorThread(_control_thread);
  
    ShenandoahInitLogger::print();
  
    return JNI_OK;
  }
  
- void ShenandoahHeap::initialize_mode() {
+ void ShenandoahHeap::initialize_generations() {
+   size_t max_capacity_new      = young_generation_capacity(max_capacity());
+   size_t soft_max_capacity_new = young_generation_capacity(soft_max_capacity());
+   size_t max_capacity_old      = max_capacity() - max_capacity_new;
+   size_t soft_max_capacity_old = soft_max_capacity() - soft_max_capacity_new;
+ 
+   _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_new, soft_max_capacity_new);
+   _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, soft_max_capacity_old);
+   _global_generation = new ShenandoahGlobalGeneration(_max_workers);
+ }
+ 
+ void ShenandoahHeap::initialize_heuristics() {
    if (ShenandoahGCMode != NULL) {
      if (strcmp(ShenandoahGCMode, "satb") == 0) {
        _gc_mode = new ShenandoahSATBMode();
      } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
        _gc_mode = new ShenandoahIUMode();
      } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
        _gc_mode = new ShenandoahPassiveMode();
+     } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
+       _gc_mode = new ShenandoahGenerationalMode();
      } else {
        vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
      }
    } else {
      ShouldNotReachHere();

@@ -427,127 +482,74 @@
    if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
      vm_exit_during_initialization(
              err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
                      _gc_mode->name()));
    }
- }
  
- void ShenandoahHeap::initialize_heuristics() {
-   assert(_gc_mode != NULL, "Must be initialized");
-   _heuristics = _gc_mode->initialize_heuristics();
- 
-   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
-     vm_exit_during_initialization(
-             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
-                     _heuristics->name()));
-   }
-   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
-     vm_exit_during_initialization(
-             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
-                     _heuristics->name()));
+   _global_generation->initialize_heuristics(_gc_mode);
+   if (mode()->is_generational()) {
+     _young_generation->initialize_heuristics(_gc_mode);
+     _old_generation->initialize_heuristics(_gc_mode);
    }
  }
  
  #ifdef _MSC_VER
  #pragma warning( push )
  #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
  #endif
  
  ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
    CollectedHeap(),
+   _gc_generation(NULL),
+   _mixed_evac(false),
+   _prep_for_mixed_evac_in_progress(false),
    _initial_size(0),
    _used(0),
    _committed(0),
-   _bytes_allocated_since_gc_start(0),
-   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
+   _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),
    _workers(NULL),
    _safepoint_workers(NULL),
    _heap_region_special(false),
    _num_regions(0),
    _regions(NULL),
    _update_refs_iterator(this),
+   _cancel_requested_time(0),
+   _young_generation(NULL),
+   _global_generation(NULL),
+   _old_generation(NULL),
    _control_thread(NULL),
+   _regulator_thread(NULL),
    _shenandoah_policy(policy),
-   _gc_mode(NULL),
-   _heuristics(NULL),
    _free_set(NULL),
    _pacer(NULL),
    _verifier(NULL),
    _phase_timings(NULL),
    _monitoring_support(NULL),
    _memory_pool(NULL),
+   _young_gen_memory_pool(NULL),
+   _old_gen_memory_pool(NULL),
    _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
    _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
    _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
    _soft_ref_policy(),
    _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
-   _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
    _marking_context(NULL),
    _bitmap_size(0),
    _bitmap_regions_per_slice(0),
    _bitmap_bytes_per_slice(0),
    _bitmap_region_special(false),
    _aux_bitmap_region_special(false),
    _liveness_cache(NULL),
-   _collection_set(NULL)
+   _collection_set(NULL),
+   _card_scan(NULL)
  {
-   // Initialize GC mode early, so we can adjust barrier support
-   initialize_mode();
-   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
- 
-   _max_workers = MAX2(_max_workers, 1U);
-   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
-                             /* are_GC_task_threads */ true,
-                             /* are_ConcurrentGC_threads */ true);
-   if (_workers == NULL) {
-     vm_exit_during_initialization("Failed necessary allocation.");
-   } else {
-     _workers->initialize_workers();
-   }
- 
-   if (ParallelGCThreads > 1) {
-     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
-                                                 ParallelGCThreads,
-                       /* are_GC_task_threads */ false,
-                  /* are_ConcurrentGC_threads */ false);
-     _safepoint_workers->initialize_workers();
-   }
  }
  
  #ifdef _MSC_VER
  #pragma warning( pop )
  #endif
  
- class ShenandoahResetBitmapTask : public AbstractGangTask {
- private:
-   ShenandoahRegionIterator _regions;
- 
- public:
-   ShenandoahResetBitmapTask() :
-     AbstractGangTask("Shenandoah Reset Bitmap") {}
- 
-   void work(uint worker_id) {
-     ShenandoahHeapRegion* region = _regions.next();
-     ShenandoahHeap* heap = ShenandoahHeap::heap();
-     ShenandoahMarkingContext* const ctx = heap->marking_context();
-     while (region != NULL) {
-       if (heap->is_bitmap_slice_committed(region)) {
-         ctx->clear_bitmap(region);
-       }
-       region = _regions.next();
-     }
-   }
- };
- 
- void ShenandoahHeap::reset_mark_bitmap() {
-   assert_gc_workers(_workers->active_workers());
-   mark_incomplete_marking_context();
- 
-   ShenandoahResetBitmapTask task;
-   _workers->run_task(&task);
- }
- 
  void ShenandoahHeap::print_on(outputStream* st) const {
    st->print_cr("Shenandoah Heap");
    st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
                 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
                 byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),

@@ -558,11 +560,12 @@
                 byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
                 proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
  
    st->print("Status: ");
    if (has_forwarded_objects())                 st->print("has forwarded objects, ");
-   if (is_concurrent_mark_in_progress())        st->print("marking, ");
+   if (is_concurrent_old_mark_in_progress())    st->print("old marking, ");
+   if (is_concurrent_young_mark_in_progress())  st->print("young marking, ");
    if (is_evacuation_in_progress())             st->print("evacuating, ");
    if (is_update_refs_in_progress())            st->print("updating refs, ");
    if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
    if (is_full_gc_in_progress())                st->print("full gc, ");
    if (is_full_gc_move_in_progress())           st->print("full gc move, ");

@@ -621,15 +624,52 @@
    if (_safepoint_workers != NULL) {
      _safepoint_workers->threads_do(&init_gclabs);
      _safepoint_workers->set_initialize_gclab();
    }
  
-   _heuristics->initialize();
- 
    JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
  }
  
+ 
+ ShenandoahOldHeuristics* ShenandoahHeap::old_heuristics() {
+   return (ShenandoahOldHeuristics*) _old_generation->heuristics();
+ }
+ 
+ bool ShenandoahHeap::doing_mixed_evacuations() {
+   return old_heuristics()->unprocessed_old_collection_candidates() > 0;
+ }
+ 
+ bool ShenandoahHeap::is_gc_generation_young() const {
+   return _gc_generation != NULL && _gc_generation->generation_mode() == YOUNG;
+ }
+ 
+ // There are three JVM parameters for setting young gen capacity:
+ //    NewSize, MaxNewSize, NewRatio.
+ //
+ // If only NewSize is set, it assigns a fixed size and the other two parameters are ignored.
+ // Otherwise NewRatio applies.
+ //
+ // If NewSize is set in any combination, it provides a lower bound.
+ //
+ // If MaxNewSize is set it provides an upper bound.
+ // If this bound is smaller than NewSize, it supersedes,
+ // resulting in a fixed size given by MaxNewSize.
+ size_t ShenandoahHeap::young_generation_capacity(size_t capacity) {
+   if (FLAG_IS_CMDLINE(NewSize) && !FLAG_IS_CMDLINE(MaxNewSize) && !FLAG_IS_CMDLINE(NewRatio)) {
+     capacity = MIN2(NewSize, capacity);
+   } else {
+     capacity /= NewRatio + 1;
+     if (FLAG_IS_CMDLINE(NewSize)) {
+       capacity = MAX2(NewSize, capacity);
+     }
+     if (FLAG_IS_CMDLINE(MaxNewSize)) {
+       capacity = MIN2(MaxNewSize, capacity);
+     }
+   }
+   return capacity;
+ }
+ 
  size_t ShenandoahHeap::used() const {
    return Atomic::load(&_used);
  }
  
  size_t ShenandoahHeap::committed() const {

@@ -657,20 +697,16 @@
  void ShenandoahHeap::decrease_used(size_t bytes) {
    assert(used() >= bytes, "never decrease heap size by more than we've left");
    Atomic::sub(&_used, bytes, memory_order_relaxed);
  }
  
- void ShenandoahHeap::increase_allocated(size_t bytes) {
-   Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
- }
- 
  void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
    size_t bytes = words * HeapWordSize;
    if (!waste) {
      increase_used(bytes);
    }
-   increase_allocated(bytes);
+ 
    if (ShenandoahPacing) {
      control_thread()->pacing_notify_alloc(words);
      if (waste) {
        pacer()->claim_for_alloc(words, true);
      }

@@ -696,10 +732,17 @@
  void ShenandoahHeap::set_soft_max_capacity(size_t v) {
    assert(min_capacity() <= v && v <= max_capacity(),
           "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
           min_capacity(), v, max_capacity());
    Atomic::store(&_soft_max_size, v);
+ 
+   if (mode()->is_generational()) {
+     size_t soft_max_capacity_young = young_generation_capacity(_soft_max_size);
+     size_t soft_max_capacity_old = _soft_max_size - soft_max_capacity_young;
+     _young_generation->set_soft_max_capacity(soft_max_capacity_young);
+     _old_generation->set_soft_max_capacity(soft_max_capacity_old);
+   }
  }
  
  size_t ShenandoahHeap::min_capacity() const {
    return _minimum_size;
  }

@@ -712,10 +755,33 @@
    HeapWord* heap_base = (HeapWord*) base();
    HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
    return p >= heap_base && p < last_region_end;
  }
  
+ bool ShenandoahHeap::is_in_young(const void* p) const {
+   return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION;
+ }
+ 
+ bool ShenandoahHeap::is_in_old(const void* p) const {
+   return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION;
+ }
+ 
+ bool ShenandoahHeap::is_in_active_generation(oop obj) const {
+   if (!mode()->is_generational()) {
+     // everything is the same single generation
+     return true;
+   }
+ 
+   if (active_generation() == NULL) {
+     // no collection is happening, only expect this to be called
+     // when concurrent processing is active, but that could change
+     return false;
+   }
+ 
+   return active_generation()->contains(obj);
+ }
+ 
  void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
    assert (ShenandoahUncommit, "should be enabled");
  
    // Application allocates from the beginning of the heap, and GC allocates at
    // the end of it. It is more efficient to uncommit from the end, so that applications

@@ -739,13 +805,40 @@
      SpinPause(); // allow allocators to take the lock
    }
  
    if (count > 0) {
      control_thread()->notify_heap_changed();
+     regulator_thread()->notify_heap_changed();
+   }
+ }
+ 
+ void ShenandoahHeap::handle_old_evacuation(HeapWord* obj, size_t words, bool promotion) {
+   // Only register the copy of the object that won the evacuation race.
+   card_scan()->register_object_wo_lock(obj);
+ 
+   // Mark the entire range of the evacuated object as dirty.  At next remembered set scan,
+   // we will clear dirty bits that do not hold interesting pointers.  It's more efficient to
+   // do this in batch, in a background GC thread than to try to carefully dirty only cards
+   // that hold interesting pointers right now.
+   card_scan()->mark_range_as_dirty(obj, words);
+ 
+   if (promotion) {
+     // This evacuation was a promotion, track this as allocation against old gen
+     old_generation()->increase_allocated(words * HeapWordSize);
+   }
+ }
+ 
+ void ShenandoahHeap::handle_old_evacuation_failure() {
+   if (_old_gen_oom_evac.try_set()) {
+     log_info(gc)("Old gen evac failure.");
    }
  }
  
+ void ShenandoahHeap::handle_promotion_failure() {
+   old_heuristics()->handle_promotion_failure();
+ }
+ 
  HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
    // New object should fit the GCLAB size
    size_t min_size = MAX2(size, PLAB::min_size());
  
    // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.

@@ -791,10 +884,115 @@
    }
    gclab->set_buf(gclab_buf, actual_size);
    return gclab->allocate(size);
  }
  
+ // Establish a new PLAB and allocate size HeapWords within it.
+ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) {
+   // New object should fit the PLAB size
+   size_t min_size = MAX2(size, PLAB::min_size());
+ 
+   // Figure out size of new PLAB, looking back at heuristics. Expand aggressively.
+   size_t new_size = ShenandoahThreadLocalData::plab_size(thread) * 2;
+   new_size = MIN2(new_size, PLAB::max_size());
+   new_size = MAX2(new_size, PLAB::min_size());
+ 
+   size_t unalignment = new_size % CardTable::card_size_in_words;
+   if (unalignment != 0) {
+     new_size = new_size - unalignment + CardTable::card_size_in_words;
+   }
+ 
+   // Record new heuristic value even if we take any shortcut. This captures
+   // the case when moderately-sized objects always take a shortcut. At some point,
+   // heuristics should catch up with them.  Note that the requested new_size may
+   // not be honored, but we remember that this is the preferred size.
+   ShenandoahThreadLocalData::set_plab_size(thread, new_size);
+ 
+   if (new_size < size) {
+     // New size still does not fit the object. Fall back to shared allocation.
+     // This avoids retiring perfectly good PLABs, when we encounter a large object.
+     return NULL;
+   }
+ 
+   // Retire current PLAB, and allocate a new one.
+   PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+   // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock.  This
+   // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is
+   // aligned with the start of a card's memory range.
+   retire_plab(plab);
+ 
+   size_t actual_size = 0;
+   HeapWord* plab_buf = allocate_new_plab(min_size, new_size, &actual_size);
+   if (plab_buf == NULL) {
+     return NULL;
+   }
+ 
+   assert (size <= actual_size, "allocation should fit");
+ 
+   if (ZeroTLAB) {
+     // ..and clear it.
+     Copy::zero_to_words(plab_buf, actual_size);
+   } else {
+     // ...and zap just allocated object.
+ #ifdef ASSERT
+     // Skip mangling the space corresponding to the object header to
+     // ensure that the returned space is not considered parsable by
+     // any concurrent GC thread.
+     size_t hdr_size = oopDesc::header_size();
+     Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
+ #endif // ASSERT
+   }
+   plab->set_buf(plab_buf, actual_size);
+   return plab->allocate(size);
+ }
+ 
+ void ShenandoahHeap::retire_plab(PLAB* plab) {
+   if (!mode()->is_generational()) {
+     plab->retire();
+   } else {
+     size_t waste = plab->waste();
+     HeapWord* top = plab->top();
+     plab->retire();
+     if (top != NULL && plab->waste() > waste && is_in_old(top)) {
+       // If retiring the plab created a filler object, then we
+       // need to register it with our card scanner so it can
+       // safely walk the region backing the plab.
+       log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT,
+                     plab->waste() - waste, p2i(top));
+       card_scan()->register_object_wo_lock(top);
+     }
+   }
+ }
+ 
+ void ShenandoahHeap::cancel_mixed_collections() {
+   assert(_old_generation != NULL, "Should only have mixed collections in generation mode.");
+   old_heuristics()->abandon_collection_candidates();
+ }
+ 
+ void ShenandoahHeap::coalesce_and_fill_old_regions() {
+   class ShenandoahGlobalCoalesceAndFill : public ShenandoahHeapRegionClosure {
+    public:
+     virtual void heap_region_do(ShenandoahHeapRegion* region) override {
+       // old region is not in the collection set and was not immediately trashed
+       if (region->is_old() && region->is_active() && !region->is_humongous()) {
+         // Reset the coalesce and fill boundary because this is a global collect
+         // and cannot be preempted by young collects. We want to be sure the entire
+         // region is coalesced here and does not resume from a previously interrupted
+         // or completed coalescing.
+         region->begin_preemptible_coalesce_and_fill();
+         region->oop_fill_and_coalesce();
+       }
+     }
+ 
+     virtual bool is_thread_safe() override {
+       return true;
+     }
+   };
+   ShenandoahGlobalCoalesceAndFill coalesce;
+   parallel_heap_region_iterate(&coalesce);
+ }
+ 
  HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
                                              size_t requested_size,
                                              size_t* actual_size) {
    ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
    HeapWord* res = allocate_memory(req);

@@ -817,10 +1015,23 @@
      *actual_size = 0;
    }
    return res;
  }
  
+ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size,
+                                             size_t word_size,
+                                             size_t* actual_size) {
+   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size);
+   HeapWord* res = allocate_memory(req);
+   if (res != NULL) {
+     *actual_size = req.actual_size();
+   } else {
+     *actual_size = 0;
+   }
+   return res;
+ }
+ 
  HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
    intptr_t pacer_epoch = 0;
    bool in_new_region = false;
    HeapWord* result = NULL;
  

@@ -865,40 +1076,69 @@
      // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
    }
  
    if (in_new_region) {
      control_thread()->notify_heap_changed();
+     regulator_thread()->notify_heap_changed();
    }
  
    if (result != NULL) {
+     ShenandoahGeneration* alloc_generation = generation_for(req.affiliation());
      size_t requested = req.size();
      size_t actual = req.actual_size();
+     size_t actual_bytes = actual * HeapWordSize;
  
      assert (req.is_lab_alloc() || (requested == actual),
              "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
              ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
  
      if (req.is_mutator_alloc()) {
        notify_mutator_alloc_words(actual, false);
+       alloc_generation->increase_allocated(actual_bytes);
  
        // If we requested more than we were granted, give the rest back to pacer.
        // This only matters if we are in the same pacing epoch: do not try to unpace
        // over the budget for the other phase.
        if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
          pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
        }
      } else {
-       increase_used(actual*HeapWordSize);
+       increase_used(actual_bytes);
      }
    }
  
    return result;
  }
  
  HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
+   if (mode()->is_generational() && req.affiliation() == YOUNG_GENERATION && young_generation()->used() + req.size() >= young_generation()->max_capacity()) {
+     return nullptr;
+   }
+ 
    ShenandoahHeapLocker locker(lock());
-   return _free_set->allocate(req, in_new_region);
+   HeapWord* result = _free_set->allocate(req, in_new_region);
+   if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) {
+     // Register the newly allocated object while we're holding the global lock since there's no synchronization
+     // built in to the implementation of register_object().  There are potential races when multiple independent
+     // threads are allocating objects, some of which might span the same card region.  For example, consider
+     // a card table's memory region within which three objects are being allocated by three different threads:
+     //
+     // objects being "concurrently" allocated:
+     //    [-----a------][-----b-----][--------------c------------------]
+     //            [---- card table memory range --------------]
+     //
+     // Before any objects are allocated, this card's memory range holds no objects.  Note that:
+     //   allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region.
+     //   allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region.
+     //   allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region.
+     //
+     // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start
+     // representing object b while first-start represents object c.  This is why we need to require all register_object()
+     // invocations to be "mutually exclusive" with respect to each card's memory range.
+     ShenandoahHeap::heap()->card_scan()->register_object(result);
+   }
+   return result;
  }
  
  HeapWord* ShenandoahHeap::mem_allocate(size_t size,
                                          bool*  gc_overhead_limit_was_exceeded) {
    ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);

@@ -909,12 +1149,12 @@
                                                               size_t size,
                                                               Metaspace::MetadataType mdtype) {
    MetaWord* result;
  
    // Inform metaspace OOM to GC heuristics if class unloading is possible.
-   if (heuristics()->can_unload_classes()) {
-     ShenandoahHeuristics* h = heuristics();
+   ShenandoahHeuristics* h = global_generation()->heuristics();
+   if (h->can_unload_classes()) {
      h->record_metaspace_oom();
    }
  
    // Expand and retry allocation
    result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

@@ -989,26 +1229,93 @@
    void do_work() {
      ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
      ShenandoahHeapRegion* r;
      while ((r =_cs->claim_next()) != NULL) {
        assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
+ 
        _sh->marked_object_iterate(r, &cl);
  
        if (ShenandoahPacing) {
          _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
        }
+       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
+         break;
+       }
+     }
+   }
+ };
+ 
+ // Unlike ShenandoahEvacuationTask, this iterates over all regions rather than just the collection set.
+ // This is needed in order to promote humongous start regions if age() >= tenure threshold.
+ class ShenandoahGenerationalEvacuationTask : public AbstractGangTask {
+ private:
+   ShenandoahHeap* const _sh;
+   ShenandoahRegionIterator *_regions;
+   bool _concurrent;
+ public:
+   ShenandoahGenerationalEvacuationTask(ShenandoahHeap* sh,
+                                        ShenandoahRegionIterator* iterator,
+                                        bool concurrent) :
+     AbstractGangTask("Shenandoah Evacuation"),
+     _sh(sh),
+     _regions(iterator),
+     _concurrent(concurrent)
+   {}
+ 
+   void work(uint worker_id) {
+     if (_concurrent) {
+       ShenandoahConcurrentWorkerSession worker_session(worker_id);
+       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
+       ShenandoahEvacOOMScope oom_evac_scope;
+       do_work();
+     } else {
+       ShenandoahParallelWorkerSession worker_session(worker_id);
+       ShenandoahEvacOOMScope oom_evac_scope;
+       do_work();
+     }
+   }
+ 
+ private:
+   void do_work() {
+     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
+     ShenandoahHeapRegion* r;
+     while ((r = _regions->next()) != nullptr) {
+       log_debug(gc)("GenerationalEvacuationTask do_work(), looking at %s region " SIZE_FORMAT ", (age: %d) [%s, %s]",
+                     r->is_old()? "old": r->is_young()? "young": "free", r->index(), r->age(),
+                     r->is_active()? "active": "inactive",
+                     r->is_humongous()? (r->is_humongous_start()? "humongous_start": "humongous_continuation"): "regular");
+       if (r->is_cset()) {
+         assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
+         _sh->marked_object_iterate(r, &cl);
+         if (ShenandoahPacing) {
+           _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
+         }
+       } else if (r->is_young() && r->is_active() && r->is_humongous_start() && (r->age() > InitialTenuringThreshold)) {
+         // We promote humongous_start regions along with their affiliated continuations during evacuation rather than
+         // doing this work during a safepoint.  We cannot put humongous regions into the collection set because that
+         // triggers the load-reference barrier (LRB) to copy on reference fetch.
+         r->promote_humongous();
+       }
+       // else, region is free, or OLD, or not in collection set, or humongous_continuation,
+       // or is young humongous_start that is too young to be promoted
  
        if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
          break;
        }
      }
    }
  };
  
  void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
-   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
-   workers()->run_task(&task);
+   if (ShenandoahHeap::heap()->mode()->is_generational()) {
+     ShenandoahRegionIterator regions;
+     ShenandoahGenerationalEvacuationTask task(this, &regions, concurrent);
+     workers()->run_task(&task);
+   } else {
+     ShenandoahEvacuationTask task(this, _collection_set, concurrent);
+     workers()->run_task(&task);
+   }
  }
  
  void ShenandoahHeap::trash_cset_regions() {
    ShenandoahHeapLocker locker(lock());
  

@@ -1031,11 +1338,11 @@
    for (size_t i = 0; i < num_regions(); i++) {
      get_region(i)->print_on(st);
    }
  }
  
- void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
+ size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
    assert(start->is_humongous_start(), "reclaim regions starting with the first one");
  
    oop humongous_obj = cast_to_oop(start->bottom());
    size_t size = humongous_obj->size();
    size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);

@@ -1051,19 +1358,24 @@
      assert(region->is_humongous(), "expect correct humongous start or continuation");
      assert(!region->is_cset(), "Humongous region should not be in collection set");
  
      region->make_trash_immediate();
    }
+   return required_regions;
  }
  
  class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
  public:
    ShenandoahCheckCleanGCLABClosure() {}
    void do_thread(Thread* thread) {
      PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
      assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
      assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
+ 
+     PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+     assert(plab != NULL, "PLAB should be initialized for %s", thread->name());
+     assert(plab->words_remaining() == 0, "PLAB should not need retirement");
    }
  };
  
  class ShenandoahRetireGCLABClosure : public ThreadClosure {
  private:

@@ -1071,14 +1383,26 @@
  public:
    ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
    void do_thread(Thread* thread) {
      PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
      assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
-     gclab->retire();
+     ShenandoahHeap::heap()->retire_plab(gclab);
      if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
        ShenandoahThreadLocalData::set_gclab_size(thread, 0);
      }
+ 
+     PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+     assert(plab != NULL, "PLAB should be initialized for %s", thread->name());
+     // TODO; Retiring a PLAB disables it so it cannot support future allocations.  This is overkill.  For old-gen
+     // regions, the important thing is to make the memory parsable by the remembered-set scanning code that drives
+     // the update-refs processing that follows.  After the updating of old-gen references is done, it is ok to carve
+     // this remnant object into smaller pieces during the subsequent evacuation pass, as long as the PLAB is made parsable
+     // again before the next update-refs phase.
+     ShenandoahHeap::heap()->retire_plab(plab);
+     if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
+       ShenandoahThreadLocalData::set_plab_size(thread, 0);
+     }
    }
  };
  
  void ShenandoahHeap::labs_make_parsable() {
    assert(UseTLAB, "Only call with UseTLAB");

@@ -1132,10 +1456,41 @@
    if (safepoint_workers() != NULL) {
      safepoint_workers()->threads_do(&cl);
    }
  }
  
+ class ShenandoahTagGCLABClosure : public ThreadClosure {
+ public:
+   void do_thread(Thread* thread) {
+     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
+     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
+     if (gclab->words_remaining() > 0) {
+       ShenandoahHeapRegion* r = ShenandoahHeap::heap()->heap_region_containing(gclab->allocate(0));
+       r->set_young_lab_flag();
+     }
+   }
+ };
+ 
+ void ShenandoahHeap::set_young_lab_region_flags() {
+   if (!UseTLAB) {
+     return;
+   }
+   for (size_t i = 0; i < _num_regions; i++) {
+     _regions[i]->clear_young_lab_flags();
+   }
+   ShenandoahTagGCLABClosure cl;
+   workers()->threads_do(&cl);
+   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+     cl.do_thread(t);
+     ThreadLocalAllocBuffer& tlab = t->tlab();
+     if (tlab.end() != NULL) {
+       ShenandoahHeapRegion* r = heap_region_containing(tlab.start());
+       r->set_young_lab_flag();
+     }
+   }
+ }
+ 
  // Returns size in bytes
  size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
    if (ShenandoahElasticTLAB) {
      // With Elastic TLABs, return the max allowed size, and let the allocation path
      // figure out the safe size for current allocation.

@@ -1527,33 +1882,10 @@
    } else {
      heap_region_iterate(blk);
    }
  }
  
- class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
- private:
-   ShenandoahMarkingContext* const _ctx;
- public:
-   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
- 
-   void heap_region_do(ShenandoahHeapRegion* r) {
-     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
-     if (r->is_active()) {
-       // Check if region needs updating its TAMS. We have updated it already during concurrent
-       // reset, so it is very likely we don't need to do another write here.
-       if (_ctx->top_at_mark_start(r) != r->top()) {
-         _ctx->capture_top_at_mark_start(r);
-       }
-     } else {
-       assert(_ctx->top_at_mark_start(r) == r->top(),
-              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
-     }
-   }
- 
-   bool is_thread_safe() { return true; }
- };
- 
  class ShenandoahRendezvousClosure : public HandshakeClosure {
  public:
    inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
    inline void do_thread(Thread* thread) {}
  };

@@ -1565,120 +1897,21 @@
  
  void ShenandoahHeap::recycle_trash() {
    free_set()->recycle_trash();
  }
  
- class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
- private:
-   ShenandoahMarkingContext* const _ctx;
- public:
-   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
- 
-   void heap_region_do(ShenandoahHeapRegion* r) {
-     if (r->is_active()) {
-       // Reset live data and set TAMS optimistically. We would recheck these under the pause
-       // anyway to capture any updates that happened since now.
-       r->clear_live_data();
-       _ctx->capture_top_at_mark_start(r);
-     }
-   }
- 
-   bool is_thread_safe() { return true; }
- };
- 
- void ShenandoahHeap::prepare_gc() {
-   reset_mark_bitmap();
- 
-   ShenandoahResetUpdateRegionStateClosure cl;
-   parallel_heap_region_iterate(&cl);
- }
- 
- class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
- private:
-   ShenandoahMarkingContext* const _ctx;
-   ShenandoahHeapLock* const _lock;
- 
- public:
-   ShenandoahFinalMarkUpdateRegionStateClosure() :
-     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
- 
-   void heap_region_do(ShenandoahHeapRegion* r) {
-     if (r->is_active()) {
-       // All allocations past TAMS are implicitly live, adjust the region data.
-       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
-       HeapWord *tams = _ctx->top_at_mark_start(r);
-       HeapWord *top = r->top();
-       if (top > tams) {
-         r->increase_live_data_alloc_words(pointer_delta(top, tams));
-       }
- 
-       // We are about to select the collection set, make sure it knows about
-       // current pinning status. Also, this allows trashing more regions that
-       // now have their pinning status dropped.
-       if (r->is_pinned()) {
-         if (r->pin_count() == 0) {
-           ShenandoahHeapLocker locker(_lock);
-           r->make_unpinned();
-         }
-       } else {
-         if (r->pin_count() > 0) {
-           ShenandoahHeapLocker locker(_lock);
-           r->make_pinned();
-         }
-       }
- 
-       // Remember limit for updating refs. It's guaranteed that we get no
-       // from-space-refs written from here on.
-       r->set_update_watermark_at_safepoint(r->top());
-     } else {
-       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
-       assert(_ctx->top_at_mark_start(r) == r->top(),
-              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
-     }
-   }
- 
-   bool is_thread_safe() { return true; }
- };
- 
- void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
-   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
-   {
-     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
-                                          ShenandoahPhaseTimings::degen_gc_final_update_region_states);
-     ShenandoahFinalMarkUpdateRegionStateClosure cl;
-     parallel_heap_region_iterate(&cl);
- 
-     assert_pinned_region_status();
-   }
- 
-   {
-     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
-                                          ShenandoahPhaseTimings::degen_gc_choose_cset);
-     ShenandoahHeapLocker locker(lock());
-     _collection_set->clear();
-     heuristics()->choose_collection_set(_collection_set);
-   }
- 
-   {
-     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
-                                          ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
-     ShenandoahHeapLocker locker(lock());
-     _free_set->rebuild();
-   }
- }
- 
  void ShenandoahHeap::do_class_unloading() {
    _unloader.unload();
  }
  
  void ShenandoahHeap::stw_weak_refs(bool full_gc) {
    // Weak refs processing
    ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
                                                  : ShenandoahPhaseTimings::degen_gc_weakrefs;
    ShenandoahTimingsTracker t(phase);
    ShenandoahGCWorkerPhase worker_phase(phase);
-   ref_processor()->process_references(phase, workers(), false /* concurrent */);
+   active_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
  }
  
  void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
  

@@ -1705,14 +1938,58 @@
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
    _gc_state.set_cond(mask, value);
    set_gc_state_all_threads(_gc_state.raw_value());
  }
  
- void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
-   assert(!has_forwarded_objects(), "Not expected before/after mark phase");
-   set_gc_state_mask(MARKING, in_progress);
-   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
+ void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
+   if (has_forwarded_objects()) {
+     set_gc_state_mask(YOUNG_MARKING | UPDATEREFS, in_progress);
+   } else {
+     set_gc_state_mask(YOUNG_MARKING, in_progress);
+   }
+ 
+   manage_satb_barrier(in_progress);
+ }
+ 
+ void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
+   if (has_forwarded_objects()) {
+     set_gc_state_mask(OLD_MARKING | UPDATEREFS, in_progress);
+   } else {
+     set_gc_state_mask(OLD_MARKING, in_progress);
+   }
+ 
+   manage_satb_barrier(in_progress);
+ }
+ 
+ void ShenandoahHeap::set_concurrent_prep_for_mixed_evacuation_in_progress(bool in_progress) {
+   // Unlike other set-gc-state functions, this may happen outside safepoint.
+   // Is only set and queried by control thread, so no coherence issues.
+   _prep_for_mixed_evac_in_progress = in_progress;
+ }
+ 
+ bool ShenandoahHeap::is_concurrent_prep_for_mixed_evacuation_in_progress() {
+   return _prep_for_mixed_evac_in_progress;
+ }
+ 
+ void ShenandoahHeap::set_aging_cycle(bool in_progress) {
+   _is_aging_cycle.set_cond(in_progress);
+ }
+ 
+ void ShenandoahHeap::manage_satb_barrier(bool active) {
+   if (is_concurrent_mark_in_progress()) {
+     // Ignore request to deactivate barrier while concurrent mark is in progress.
+     // Do not attempt to re-activate the barrier if it is already active.
+     if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
+       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
+     }
+   } else {
+     // No concurrent marking is in progress so honor request to deactivate,
+     // but only if the barrier is already active.
+     if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
+       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
+     }
+   }
  }
  
  void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
    set_gc_state_mask(EVACUATION, in_progress);

@@ -1753,25 +2030,37 @@
        SpinPause();
      }
    }
  }
  
+ void ShenandoahHeap::cancel_concurrent_mark() {
+   _young_generation->cancel_marking();
+   _old_generation->cancel_marking();
+   _global_generation->cancel_marking();
+ 
+   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
+ }
+ 
  void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
    if (try_cancel_gc()) {
      FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
      log_info(gc)("%s", msg.buffer());
      Events::log(Thread::current(), "%s", msg.buffer());
+     _cancel_requested_time = os::elapsedTime();
    }
  }
  
  uint ShenandoahHeap::max_workers() {
    return _max_workers;
  }
  
  void ShenandoahHeap::stop() {
    // The shutdown sequence should be able to terminate when GC is running.
  
+   // Step 0a. Stop requesting collections.
+   regulator_thread()->stop();
+ 
    // Step 0. Notify policy to disable event recording.
    _shenandoah_policy->record_shutdown();
  
    // Step 1. Notify control thread that we are in shutdown.
    // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.

@@ -1872,24 +2161,21 @@
    ShenandoahHeap* heap = ShenandoahHeap::heap();
    assert(heap->collection_set() != NULL, "Sanity");
    return (address) heap->collection_set()->biased_map_address();
  }
  
- address ShenandoahHeap::cancelled_gc_addr() {
-   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
- }
- 
  address ShenandoahHeap::gc_state_addr() {
    return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
  }
  
- size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
-   return Atomic::load(&_bytes_allocated_since_gc_start);
- }
- 
  void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
-   Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
+   if (mode()->is_generational()) {
+     young_generation()->reset_bytes_allocated_since_gc_start();
+     old_generation()->reset_bytes_allocated_since_gc_start();
+   }
+ 
+   global_generation()->reset_bytes_allocated_since_gc_start();
  }
  
  void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
    _degenerated_gc_in_progress.set_cond(in_progress);
  }

@@ -1954,12 +2240,14 @@
  
  #ifdef ASSERT
  void ShenandoahHeap::assert_pinned_region_status() {
    for (size_t i = 0; i < num_regions(); i++) {
      ShenandoahHeapRegion* r = get_region(i);
-     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
-            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+     if (active_generation()->contains(r)) {
+       assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
+              "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+     }
    }
  }
  #endif
  
  ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {

@@ -2015,39 +2303,115 @@
  template<bool CONCURRENT>
  class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
  private:
    ShenandoahHeap* _heap;
    ShenandoahRegionIterator* _regions;
+   bool _mixed_evac;             // true iff most recent evacuation includes old-gen HeapRegions
+ 
  public:
-   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
+   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool mixed_evac) :
      AbstractGangTask("Shenandoah Update References"),
      _heap(ShenandoahHeap::heap()),
-     _regions(regions) {
+     _regions(regions),
+     _mixed_evac(mixed_evac)
+   {
    }
  
    void work(uint worker_id) {
      if (CONCURRENT) {
        ShenandoahConcurrentWorkerSession worker_session(worker_id);
        ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
-       do_work<ShenandoahConcUpdateRefsClosure>();
+       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
      } else {
        ShenandoahParallelWorkerSession worker_session(worker_id);
-       do_work<ShenandoahSTWUpdateRefsClosure>();
+       do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
      }
    }
  
  private:
    template<class T>
-   void do_work() {
+   void do_work(uint worker_id) {
      T cl;
      ShenandoahHeapRegion* r = _regions->next();
-     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
+ 
+     // We update references for global, old, and young collections.
+     assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking");
+     ShenandoahMarkingContext* const ctx = _heap->marking_context();
+     bool is_mixed = _heap->collection_set()->has_old_regions();
      while (r != NULL) {
        HeapWord* update_watermark = r->get_update_watermark();
        assert (update_watermark >= r->bottom(), "sanity");
+ 
+       log_debug(gc)("ShenandoahUpdateHeapRefsTask::do_work(%u) looking at region " SIZE_FORMAT, worker_id, r->index());
        if (r->is_active() && !r->is_cset()) {
-         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
+         if (!_heap->mode()->is_generational() || (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION)) {
+           _heap->marked_object_oop_iterate(r, &cl, update_watermark);
+         } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) {
+           if (_heap->active_generation()->generation_mode() == GLOBAL) {
+             _heap->marked_object_oop_iterate(r, &cl, update_watermark);
+           } else {
+             // Old region in a young cycle or mixed cycle.
+             if (!_mixed_evac) {
+               // This is a young evac..
+               _heap->card_scan()->process_region(r, &cl, true);
+             } else {
+               // This is a _mixed_evac.
+               //
+               // TODO: For _mixed_evac, consider building an old-gen remembered set that allows restricted updating
+               // within old-gen HeapRegions.  This remembered set can be constructed by old-gen concurrent marking
+               // and augmented by card marking.  For example, old-gen concurrent marking can remember for each old-gen
+               // card which other old-gen regions it refers to: none, one-other specifically, multiple-other non-specific.
+               // Update-references when _mixed_evac processess each old-gen memory range that has a traditional DIRTY
+               // card or if the "old-gen remembered set" indicates that this card holds pointers specifically to an
+               // old-gen region in the most recent collection set, or if this card holds pointers to other non-specific
+               // old-gen heap regions.
+               if (r->is_humongous()) {
+                 r->oop_iterate_humongous(&cl);
+               } else {
+                 // This is a mixed evacuation.  Old regions that are candidates for collection have not been coalesced
+                 // and filled.  Use mark bits to find objects that need to be updated.
+                 //
+                 // Future TODO: establish a second remembered set to identify which old-gen regions point to other old-gen
+                 // regions which are in the collection set for a particular mixed evacuation.
+                 HeapWord *p = r->bottom();
+                 ShenandoahObjectToOopBoundedClosure<T> objs(&cl, p, update_watermark);
+ 
+                 // Anything beyond update_watermark was allocated during evacuation.  Thus, it is known to not hold
+                 // references to collection set objects.
+                 while (p < update_watermark) {
+                   oop obj = cast_to_oop(p);
+                   if (ctx->is_marked(obj)) {
+                     objs.do_object(obj);
+                     p += obj->size();
+                   } else {
+                     // This object is not marked so we don't scan it.
+                     HeapWord* tams = ctx->top_at_mark_start(r);
+                     if (p >= tams) {
+                       p += obj->size();
+                     } else {
+                       p = ctx->get_next_marked_addr(p, tams);
+                     }
+                   }
+                 }
+               }
+             }
+           }
+         } else {
+           // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions
+           // to a non-free active region while this loop is executing.  Whenever this happens, the changing of a region's
+           // active status may propagate at a different speed than the changing of the region's affiliation.
+ 
+           // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen
+           // by this thread before the region's affiliation() is seen by this thread.
+ 
+           // It's ok for this race to occur because the newly transformed region does not have any references to be
+           // updated.
+ 
+           assert(r->get_update_watermark() == r->bottom(),
+                  "%s Region " SIZE_FORMAT " is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE",
+                  affiliation_name(r->affiliation()), r->index());
+         }
        }
        if (ShenandoahPacing) {
          _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
        }
        if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {

@@ -2060,14 +2424,14 @@
  
  void ShenandoahHeap::update_heap_references(bool concurrent) {
    assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
  
    if (concurrent) {
-     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
+     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator, _mixed_evac);
      workers()->run_task(&task);
    } else {
-     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
+     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator, _mixed_evac);
      workers()->run_task(&task);
    }
  }
  
  

@@ -2240,13 +2604,22 @@
  bool ShenandoahHeap::should_inject_alloc_failure() {
    return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
  }
  
  void ShenandoahHeap::initialize_serviceability() {
-   _memory_pool = new ShenandoahMemoryPool(this);
-   _cycle_memory_manager.add_pool(_memory_pool);
-   _stw_memory_manager.add_pool(_memory_pool);
+   if (mode()->is_generational()) {
+     _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this);
+     _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this);
+     _cycle_memory_manager.add_pool(_young_gen_memory_pool);
+     _cycle_memory_manager.add_pool(_old_gen_memory_pool);
+     _stw_memory_manager.add_pool(_young_gen_memory_pool);
+     _stw_memory_manager.add_pool(_old_gen_memory_pool);
+   } else {
+     _memory_pool = new ShenandoahMemoryPool(this);
+     _cycle_memory_manager.add_pool(_memory_pool);
+     _stw_memory_manager.add_pool(_memory_pool);
+   }
  }
  
  GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
    GrowableArray<GCMemoryManager*> memory_managers(2);
    memory_managers.append(&_cycle_memory_manager);

@@ -2254,16 +2627,21 @@
    return memory_managers;
  }
  
  GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
    GrowableArray<MemoryPool*> memory_pools(1);
-   memory_pools.append(_memory_pool);
+   if (mode()->is_generational()) {
+     memory_pools.append(_young_gen_memory_pool);
+     memory_pools.append(_old_gen_memory_pool);
+   } else {
+     memory_pools.append(_memory_pool);
+   }
    return memory_pools;
  }
  
  MemoryUsage ShenandoahHeap::memory_usage() {
-   return _memory_pool->get_memory_usage();
+   return MemoryUsage(_initial_size, used(), committed(), max_capacity());
  }
  
  ShenandoahRegionIterator::ShenandoahRegionIterator() :
    _heap(ShenandoahHeap::heap()),
    _index(0) {}

@@ -2297,14 +2675,221 @@
  
  void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
    assert(worker_id < _max_workers, "sanity");
    assert(_liveness_cache != NULL, "sanity");
    ShenandoahLiveData* ld = _liveness_cache[worker_id];
+ 
    for (uint i = 0; i < num_regions(); i++) {
      ShenandoahLiveData live = ld[i];
      if (live > 0) {
        ShenandoahHeapRegion* r = get_region(i);
        r->increase_live_data_gc_words(live);
        ld[i] = 0;
      }
    }
  }
+ 
+ void ShenandoahHeap::purge_old_satb_buffers(bool abandon) {
+   ((ShenandoahOldGeneration*)_old_generation)->purge_satb_buffers(abandon);
+ }
+ 
+ template<>
+ void ShenandoahGenerationRegionClosure<YOUNG>::heap_region_do(ShenandoahHeapRegion* region) {
+   // Visit young and free regions
+   if (region->affiliation() != OLD_GENERATION) {
+     _cl->heap_region_do(region);
+   }
+ }
+ 
+ template<>
+ void ShenandoahGenerationRegionClosure<OLD>::heap_region_do(ShenandoahHeapRegion* region) {
+   // Visit old and free regions
+   if (region->affiliation() != YOUNG_GENERATION) {
+     _cl->heap_region_do(region);
+   }
+ }
+ 
+ template<>
+ void ShenandoahGenerationRegionClosure<GLOBAL>::heap_region_do(ShenandoahHeapRegion* region) {
+   _cl->heap_region_do(region);
+ }
+ 
+ // Assure that the remember set has a dirty card everywhere there is an interesting pointer.
+ // This examines the read_card_table between bottom() and top() since all PLABS are retired
+ // before the safepoint for init_mark.  Actually, we retire them before update-references and don't
+ // restore them until the start of evacuation.
+ void ShenandoahHeap::verify_rem_set_at_mark() {
+   shenandoah_assert_safepoint();
+   assert(mode()->is_generational(), "Only verify remembered set for generational operational modes");
+ 
+   ShenandoahRegionIterator iterator;
+   RememberedScanner* scanner = card_scan();
+   ShenandoahVerifyRemSetClosure check_interesting_pointers(true);
+   ShenandoahMarkingContext* ctx;
+ 
+   log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young");
+ 
+   if (doing_mixed_evacuations() ||
+       is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) {
+     ctx = complete_marking_context();
+   } else {
+     ctx = nullptr;
+   }
+ 
+   while (iterator.has_next()) {
+     ShenandoahHeapRegion* r = iterator.next();
+     if (r == nullptr)
+       break;
+     if (r->is_old() && r->is_active()) {
+       HeapWord* obj_addr = r->bottom();
+       if (r->is_humongous_start()) {
+         oop obj = cast_to_oop(obj_addr);
+         if (!ctx || ctx->is_marked(obj)) {
+           // For humongous objects, the typical object is an array, so the following checks may be overkill
+           // For regular objects (not object arrays), if the card holding the start of the object is dirty,
+           // we do not need to verify that cards spanning interesting pointers within this object are dirty.
+           if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) {
+             obj->oop_iterate(&check_interesting_pointers);
+           }
+           // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered
+         }
+         // else, this humongous object is not marked so no need to verify its internal pointers
+         if (!scanner->verify_registration(obj_addr, ctx)) {
+           ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL,
+                                           "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__);
+         }
+       } else if (!r->is_humongous()) {
+         HeapWord* top = r->top();
+         while (obj_addr < top) {
+           oop obj = cast_to_oop(obj_addr);
+           // ctx->is_marked() returns true if mark bit set (TAMS not relevant during init mark)
+           if (!ctx || ctx->is_marked(obj)) {
+             // For regular objects (not object arrays), if the card holding the start of the object is dirty,
+             // we do not need to verify that cards spanning interesting pointers within this object are dirty.
+             if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) {
+               obj->oop_iterate(&check_interesting_pointers);
+             }
+             // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered
+             if (!scanner->verify_registration(obj_addr, ctx)) {
+               ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL,
+                                             "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__);
+             }
+             obj_addr += obj->size();
+           } else {
+             // This object is not live so we don't verify dirty cards contained therein
+             assert(ctx->top_at_mark_start(r) == top, "Expect tams == top at start of mark.");
+             obj_addr = ctx->get_next_marked_addr(obj_addr, top);
+           }
+         }
+       } // else, we ignore humongous continuation region
+     } // else, this is not an OLD region so we ignore it
+   } // all regions have been processed
+ }
+ 
+ void ShenandoahHeap::help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, HeapWord* from,
+                                                 HeapWord* top, HeapWord* registration_watermark, const char* message) {
+   RememberedScanner* scanner = card_scan();
+   ShenandoahVerifyRemSetClosure check_interesting_pointers(false);
+ 
+   HeapWord* obj_addr = from;
+   if (r->is_humongous_start()) {
+     oop obj = cast_to_oop(obj_addr);
+     if (!ctx || ctx->is_marked(obj)) {
+       size_t card_index = scanner->card_index_for_addr(obj_addr);
+       // For humongous objects, the typical object is an array, so the following checks may be overkill
+       // For regular objects (not object arrays), if the card holding the start of the object is dirty,
+       // we do not need to verify that cards spanning interesting pointers within this object are dirty.
+       if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) {
+         obj->oop_iterate(&check_interesting_pointers);
+       }
+       // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered
+     }
+     // else, this humongous object is not live so no need to verify its internal pointers
+ 
+     if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) {
+       ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message,
+                                        "object not properly registered", __FILE__, __LINE__);
+     }
+   } else if (!r->is_humongous()) {
+     while (obj_addr < top) {
+       oop obj = cast_to_oop(obj_addr);
+       // ctx->is_marked() returns true if mark bit set or if obj above TAMS.
+       if (!ctx || ctx->is_marked(obj)) {
+         size_t card_index = scanner->card_index_for_addr(obj_addr);
+         // For regular objects (not object arrays), if the card holding the start of the object is dirty,
+         // we do not need to verify that cards spanning interesting pointers within this object are dirty.
+         if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) {
+           obj->oop_iterate(&check_interesting_pointers);
+         }
+         // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered
+ 
+         if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) {
+           ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message,
+                                            "object not properly registered", __FILE__, __LINE__);
+         }
+         obj_addr += obj->size();
+       } else {
+         // This object is not live so we don't verify dirty cards contained therein
+         HeapWord* tams = ctx->top_at_mark_start(r);
+         obj_addr = ctx->get_next_marked_addr(obj_addr, tams);
+       }
+     }
+   }
+ }
+ 
+ void ShenandoahHeap::verify_rem_set_after_full_gc() {
+   shenandoah_assert_safepoint();
+   assert(mode()->is_generational(), "Only verify remembered set for generational operational modes");
+ 
+   ShenandoahRegionIterator iterator;
+ 
+   while (iterator.has_next()) {
+     ShenandoahHeapRegion* r = iterator.next();
+     if (r == nullptr)
+       break;
+     if (r->is_old() && !r->is_cset()) {
+       help_verify_region_rem_set(r, nullptr, r->bottom(), r->top(), r->top(), "Remembered set violation at end of Full GC");
+     }
+   }
+ }
+ 
+ // Assure that the remember set has a dirty card everywhere there is an interesting pointer.  Even though
+ // the update-references scan of remembered set only examines cards up to update_watermark, the remembered
+ // set should be valid through top.  This examines the write_card_table between bottom() and top() because
+ // all PLABS are retired immediately before the start of update refs.
+ void ShenandoahHeap::verify_rem_set_at_update_ref() {
+   shenandoah_assert_safepoint();
+   assert(mode()->is_generational(), "Only verify remembered set for generational operational modes");
+ 
+   ShenandoahRegionIterator iterator;
+   ShenandoahMarkingContext* ctx;
+ 
+   if (doing_mixed_evacuations() ||
+       is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) {
+     ctx = complete_marking_context();
+   } else {
+     ctx = nullptr;
+   }
+ 
+   while (iterator.has_next()) {
+     ShenandoahHeapRegion* r = iterator.next();
+     if (r == nullptr)
+       break;
+     if (r->is_old() && !r->is_cset()) {
+       help_verify_region_rem_set(r, ctx, r->bottom(), r->top(), r->get_update_watermark(),
+                                  "Remembered set violation at init-update-references");
+     }
+   }
+ }
+ 
+ ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahRegionAffiliation affiliation) const {
+   if (!mode()->is_generational()) {
+     return global_generation();
+   } else if (affiliation == YOUNG_GENERATION) {
+     return young_generation();
+   } else if (affiliation == OLD_GENERATION) {
+     return old_generation();
+   }
+ 
+   ShouldNotReachHere();
+   return nullptr;
+ }
< prev index next >