src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

*** 28,39 ***
  #include "gc/g1/g1FullCollector.inline.hpp"
  #include "gc/g1/g1FullGCCompactionPoint.hpp"
  #include "gc/g1/g1FullGCCompactTask.hpp"
  #include "gc/g1/heapRegion.inline.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
  #include "logging/log.hpp"
  #include "oops/oop.inline.hpp"
  #include "utilities/ticks.hpp"
  
! void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
    assert(_bitmap->is_marked(obj), "Should only compact marked objects");
    _bitmap->clear(obj);
  }
  
! size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
    size_t size = obj->size();
!   if (obj->is_forwarded()) {
!     G1FullGCCompactTask::copy_object_to_new_location(obj);
    }
  
    // Clear the mark for the compacted object to allow reuse of the
    // bitmap without an additional clearing step.
    clear_in_bitmap(obj);
    return size;
  }
  
  void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
!   assert(obj->is_forwarded(), "Sanity!");
!   assert(obj->forwardee() != obj, "Object must have a new location");
  
    size_t size = obj->size();
    // Copy object and reinit its mark.
    HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
!   HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
    Copy::aligned_conjoint_words(obj_addr, destination, size);
  
    // There is no need to transform stack chunks - marking already did that.
    cast_to_oop(destination)->init_mark();
    assert(cast_to_oop(destination)->klass() != nullptr, "should have a class");
--- 28,43 ---
  #include "gc/g1/g1FullCollector.inline.hpp"
  #include "gc/g1/g1FullGCCompactionPoint.hpp"
  #include "gc/g1/g1FullGCCompactTask.hpp"
  #include "gc/g1/heapRegion.inline.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
+ #include "gc/shared/slidingForwarding.inline.hpp"
  #include "logging/log.hpp"
  #include "oops/oop.inline.hpp"
  #include "utilities/ticks.hpp"
  
! template <bool ALT_FWD>
+ void G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::clear_in_bitmap(oop obj) {
    assert(_bitmap->is_marked(obj), "Should only compact marked objects");
    _bitmap->clear(obj);
  }
  
! template <bool ALT_FWD>
+ size_t G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::apply(oop obj) {
    size_t size = obj->size();
!   if (SlidingForwarding::is_forwarded(obj)) {
!     G1FullGCCompactTask::copy_object_to_new_location<ALT_FWD>(obj);
    }
  
    // Clear the mark for the compacted object to allow reuse of the
    // bitmap without an additional clearing step.
    clear_in_bitmap(obj);
    return size;
  }
  
+ template <bool ALT_FWD>
  void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
!   assert(SlidingForwarding::is_forwarded(obj), "Sanity!");
!   assert(SlidingForwarding::forwardee<ALT_FWD>(obj) != obj, "Object must have a new location");
  
    size_t size = obj->size();
    // Copy object and reinit its mark.
    HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
!   HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
    Copy::aligned_conjoint_words(obj_addr, destination, size);
  
    // There is no need to transform stack chunks - marking already did that.
    cast_to_oop(destination)->init_mark();
    assert(cast_to_oop(destination)->klass() != nullptr, "should have a class");

*** 75,12 ***
      // for bitmap verification and to be able to use the bitmap
      // for evacuation failures in the next young collection. Testing
      // showed that it was better overall to clear bit by bit, compared
      // to clearing the whole region at the end. This difference was
      // clearly seen for regions with few marks.
!     G1CompactRegionClosure compact(collector()->mark_bitmap());
!     hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
    }
  
    hr->reset_compacted_after_full_gc(_collector->compaction_top(hr));
  }
  
--- 79,17 ---
      // for bitmap verification and to be able to use the bitmap
      // for evacuation failures in the next young collection. Testing
      // showed that it was better overall to clear bit by bit, compared
      // to clearing the whole region at the end. This difference was
      // clearly seen for regions with few marks.
!     if (UseAltGCForwarding) {
!       G1CompactRegionClosure<true> compact(collector()->mark_bitmap());
+       hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+     } else {
+       G1CompactRegionClosure<false> compact(collector()->mark_bitmap());
+       hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+     }
    }
  
    hr->reset_compacted_after_full_gc(_collector->compaction_top(hr));
  }
  

*** 102,32 ***
         ++it) {
      compact_region(*it);
    }
  }
  
! void G1FullGCCompactTask::humongous_compaction() {
!   GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
- 
    for (HeapRegion* hr : collector()->humongous_compaction_regions()) {
      assert(collector()->is_compaction_target(hr->hrm_index()), "Sanity");
!     compact_humongous_obj(hr);
    }
  }
  
  void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
    assert(src_hr->is_starts_humongous(), "Should be start region of the humongous object");
  
    oop obj = cast_to_oop(src_hr->bottom());
    size_t word_size = obj->size();
  
    uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
!   HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
  
    assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
    collector()->mark_bitmap()->clear(obj);
  
!   copy_object_to_new_location(obj);
  
    uint dest_start_idx = _g1h->addr_to_region(destination);
    // Update the metadata for the destination regions.
    _g1h->set_humongous_metadata(_g1h->region_at(dest_start_idx), num_regions, word_size, false);
  
--- 111,41 ---
         ++it) {
      compact_region(*it);
    }
  }
  
! template <bool ALT_FWD>
! void G1FullGCCompactTask::humongous_compaction_impl() {
    for (HeapRegion* hr : collector()->humongous_compaction_regions()) {
      assert(collector()->is_compaction_target(hr->hrm_index()), "Sanity");
!     compact_humongous_obj<ALT_FWD>(hr);
+   }
+ }
+ 
+ void G1FullGCCompactTask::humongous_compaction() {
+   GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
+   if (UseAltGCForwarding) {
+     humongous_compaction_impl<true>();
+   } else {
+     humongous_compaction_impl<false>();
    }
  }
  
+ template <bool ALT_FWD>
  void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
    assert(src_hr->is_starts_humongous(), "Should be start region of the humongous object");
  
    oop obj = cast_to_oop(src_hr->bottom());
    size_t word_size = obj->size();
  
    uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
!   HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
  
    assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
    collector()->mark_bitmap()->clear(obj);
  
!   copy_object_to_new_location<ALT_FWD>(obj);
  
    uint dest_start_idx = _g1h->addr_to_region(destination);
    // Update the metadata for the destination regions.
    _g1h->set_humongous_metadata(_g1h->region_at(dest_start_idx), num_regions, word_size, false);