src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

*** 30,10 ***
--- 30,11 ---
  #include "gc/shared/classUnloadingContext.hpp"
  #include "gc/shared/fullGCForwarding.hpp"
  #include "gc/shared/gcArguments.hpp"
  #include "gc/shared/gcTimer.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
+ #include "gc/shared/gc_globals.hpp"
  #include "gc/shared/locationPrinter.inline.hpp"
  #include "gc/shared/memAllocator.hpp"
  #include "gc/shared/plab.hpp"
  #include "gc/shared/tlab_globals.hpp"
  #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"

*** 199,12 ***
    size_t num_min_regions = min_byte_size / reg_size_bytes;
    num_min_regions = MIN2(num_min_regions, _num_regions);
    assert(num_min_regions <= _num_regions, "sanity");
    _minimum_size = num_min_regions * reg_size_bytes;
  
!   // Default to max heap size.
-   _soft_max_size = _num_regions * reg_size_bytes;
  
    _committed = _initial_size;
  
    size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
    size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
--- 200,11 ---
    size_t num_min_regions = min_byte_size / reg_size_bytes;
    num_min_regions = MIN2(num_min_regions, _num_regions);
    assert(num_min_regions <= _num_regions, "sanity");
    _minimum_size = num_min_regions * reg_size_bytes;
  
!   _soft_max_size = SoftMaxHeapSize;
  
    _committed = _initial_size;
  
    size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
    size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();

*** 522,11 ***
                      _gc_mode->name()));
    }
  }
  
  void ShenandoahHeap::initialize_heuristics() {
!   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
    _global_generation->initialize_heuristics(mode());
  }
  
  #ifdef _MSC_VER
  #pragma warning( push )
--- 522,11 ---
                      _gc_mode->name()));
    }
  }
  
  void ShenandoahHeap::initialize_heuristics() {
!   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity());
    _global_generation->initialize_heuristics(mode());
  }
  
  #ifdef _MSC_VER
  #pragma warning( push )

*** 1236,10 ***
--- 1236,15 ---
  
        // There are two reasons to retire all plabs between old-gen evacuation passes.
        //  1. We need to make the plab memory parsable by remembered-set scanning.
        //  2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
        ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
+ 
+       // Re-enable promotions for the next evacuation phase.
+       ShenandoahThreadLocalData::enable_plab_promotions(thread);
+ 
+       // Reset the fill size for next evacuation phase.
        if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
          ShenandoahThreadLocalData::set_plab_size(thread, 0);
        }
      }
    }

*** 1777,16 ***
    reclaim_aux_bitmap_for_iteration();
  }
  
  bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
    assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
! 
!   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
!     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
!     return false;
    }
-   // Reset bitmap
    _aux_bit_map.clear();
    return true;
  }
  
  void ShenandoahHeap::scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops) {
--- 1782,18 ---
    reclaim_aux_bitmap_for_iteration();
  }
  
  bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
    assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
!   if (!_aux_bitmap_region_special) {
!     bool success = os::commit_memory((char *) _aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false);
!     if (!success) {
!       log_warning(gc)("Auxiliary marking bitmap commit failed: " PTR_FORMAT " (%zu bytes)",
+                       p2i(_aux_bitmap_region.start()), _aux_bitmap_region.byte_size());
+       return false;
+     }
    }
    _aux_bit_map.clear();
    return true;
  }
  
  void ShenandoahHeap::scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops) {

*** 1798,12 ***
    ShenandoahHeapIterationRootScanner rp(n_workers);
    rp.roots_do(oops);
  }
  
  void ShenandoahHeap::reclaim_aux_bitmap_for_iteration() {
!   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
!     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
    }
  }
  
  // Closure for parallelly iterate objects
  class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure {
--- 1805,17 ---
    ShenandoahHeapIterationRootScanner rp(n_workers);
    rp.roots_do(oops);
  }
  
  void ShenandoahHeap::reclaim_aux_bitmap_for_iteration() {
!   if (!_aux_bitmap_region_special) {
!     bool success = os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size());
+     if (!success) {
+       log_warning(gc)("Auxiliary marking bitmap uncommit failed: " PTR_FORMAT " (%zu bytes)",
+                       p2i(_aux_bitmap_region.start()), _aux_bitmap_region.byte_size());
+       assert(false, "Auxiliary marking bitmap uncommit should always succeed");
+     }
    }
  }
  
  // Closure for parallelly iterate objects
  class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure {

*** 2594,63 ***
      }
    }
    return false;
  }
  
! bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
    shenandoah_assert_heaplocked();
! 
-   // Bitmaps in special regions do not need commits
-   if (_bitmap_region_special) {
-     return true;
-   }
  
    if (is_bitmap_slice_committed(r, true)) {
      // Some other region from the group is already committed, meaning the bitmap
      // slice is already committed, we exit right away.
!     return true;
    }
  
    // Commit the bitmap slice:
    size_t slice = r->index() / _bitmap_regions_per_slice;
    size_t off = _bitmap_bytes_per_slice * slice;
    size_t len = _bitmap_bytes_per_slice;
    char* start = (char*) _bitmap_region.start() + off;
  
!   if (!os::commit_memory(start, len, false)) {
-     return false;
-   }
  
    if (AlwaysPreTouch) {
      os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
    }
- 
-   return true;
  }
  
! bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
    shenandoah_assert_heaplocked();
! 
-   // Bitmaps in special regions do not need uncommits
-   if (_bitmap_region_special) {
-     return true;
-   }
  
    if (is_bitmap_slice_committed(r, true)) {
      // Some other region from the group is still committed, meaning the bitmap
      // slice should stay committed, exit right away.
!     return true;
    }
  
    // Uncommit the bitmap slice:
    size_t slice = r->index() / _bitmap_regions_per_slice;
    size_t off = _bitmap_bytes_per_slice * slice;
    size_t len = _bitmap_bytes_per_slice;
!   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
!     return false;
    }
-   return true;
  }
  
  void ShenandoahHeap::forbid_uncommit() {
    if (_uncommit_thread != nullptr) {
      _uncommit_thread->forbid_uncommit();
--- 2606,54 ---
      }
    }
    return false;
  }
  
! void ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
    shenandoah_assert_heaplocked();
!   assert(!is_bitmap_region_special(), "Not for special memory");
  
    if (is_bitmap_slice_committed(r, true)) {
      // Some other region from the group is already committed, meaning the bitmap
      // slice is already committed, we exit right away.
!     return;
    }
  
    // Commit the bitmap slice:
    size_t slice = r->index() / _bitmap_regions_per_slice;
    size_t off = _bitmap_bytes_per_slice * slice;
    size_t len = _bitmap_bytes_per_slice;
    char* start = (char*) _bitmap_region.start() + off;
  
!   os::commit_memory_or_exit(start, len, false, "Unable to commit bitmap slice");
  
    if (AlwaysPreTouch) {
      os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
    }
  }
  
! void ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
    shenandoah_assert_heaplocked();
!   assert(!is_bitmap_region_special(), "Not for special memory");
  
    if (is_bitmap_slice_committed(r, true)) {
      // Some other region from the group is still committed, meaning the bitmap
      // slice should stay committed, exit right away.
!     return;
    }
  
    // Uncommit the bitmap slice:
    size_t slice = r->index() / _bitmap_regions_per_slice;
    size_t off = _bitmap_bytes_per_slice * slice;
    size_t len = _bitmap_bytes_per_slice;
! 
!   char* addr = (char*) _bitmap_region.start() + off;
+   bool success = os::uncommit_memory(addr, len);
+   if (!success) {
+     log_warning(gc)("Bitmap slice uncommit failed: " PTR_FORMAT " (%zu bytes)", p2i(addr), len);
+     assert(false, "Bitmap slice uncommit should always succeed");
    }
  }
  
  void ShenandoahHeap::forbid_uncommit() {
    if (_uncommit_thread != nullptr) {
      _uncommit_thread->forbid_uncommit();