< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

Print this page
@@ -26,10 +26,11 @@
  
  #include "compiler/oopMap.hpp"
  #include "gc/shared/continuationGCSupport.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
  #include "gc/shared/preservedMarks.inline.hpp"
+ #include "gc/shared/slidingForwarding.inline.hpp"
  #include "gc/shared/tlab_globals.hpp"
  #include "gc/shared/workerThread.hpp"
  #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  #include "gc/shenandoah/shenandoahConcurrentGC.hpp"

@@ -226,19 +227,22 @@
    {
      // The rest of code performs region moves, where region status is undefined
      // until all phases run together.
      ShenandoahHeapLocker lock(heap->lock());
  
+     SlidingForwarding::begin();
+ 
      phase2_calculate_target_addresses(worker_slices);
  
      OrderAccess::fence();
  
      phase3_update_references();
  
      phase4_compact_objects(worker_slices);
  
      phase5_epilog();
+     SlidingForwarding::end();
    }
  
    // Resize metaspace
    MetaspaceGC::compute_new_size();
  

@@ -298,10 +302,11 @@
    ShenandoahSTWMark mark(true /*full_gc*/);
    mark.mark();
    heap->parallel_cleaning(true /* full_gc */);
  }
  
+ template <bool ALT_FWD>
  class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
  private:
    PreservedMarks*          const _preserved_marks;
    ShenandoahHeap*          const _heap;
    GrowableArray<ShenandoahHeapRegion*>& _empty_regions;

@@ -366,11 +371,11 @@
  
      // Object fits into current region, record new location:
      assert(_compact_point + obj_size <= _to_region->end(), "must fit");
      shenandoah_assert_not_forwarded(nullptr, p);
      _preserved_marks->push_if_necessary(p, p->mark());
-     p->forward_to(cast_to_oop(_compact_point));
+     SlidingForwarding::forward_to<ALT_FWD>(p, cast_to_oop(_compact_point));
      _compact_point += obj_size;
    }
  };
  
  class ShenandoahPrepareForCompactionTask : public WorkerTask {

@@ -396,20 +401,31 @@
      // Can move the region, and this is not the humongous region. Humongous
      // moves are special cased here, because their moves are handled separately.
      return r->is_stw_move_allowed() && !r->is_humongous();
    }
  
-   void work(uint worker_id) override;
+   void work(uint worker_id) override {
+     if (UseAltGCForwarding) {
+       work_impl<true>(worker_id);
+     } else {
+       work_impl<false>(worker_id);
+     }
+   }
+ 
  private:
+   template<bool ALT_FWD>
+   void work_impl(uint worker_id);
+ 
    template<typename ClosureType>
    void prepare_for_compaction(ClosureType& cl,
                                GrowableArray<ShenandoahHeapRegion*>& empty_regions,
                                ShenandoahHeapRegionSetIterator& it,
                                ShenandoahHeapRegion* from_region);
  };
  
- void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
+ template<bool ALT_FWD>
+ void ShenandoahPrepareForCompactionTask::work_impl(uint worker_id) {
    ShenandoahParallelWorkerSession worker_session(worker_id);
    ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
    ShenandoahHeapRegionSetIterator it(slice);
    ShenandoahHeapRegion* from_region = it.next();
    // No work?

@@ -421,11 +437,11 @@
    // Remember empty regions and reuse them as needed.
    ResourceMark rm;
  
    GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
  
-   ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+   ShenandoahPrepareForCompactionObjectClosure<ALT_FWD> cl(_preserved_marks->get(worker_id), empty_regions, from_region);
    prepare_for_compaction(cl, empty_regions, it, from_region);
  }
  
  template<typename ClosureType>
  void ShenandoahPrepareForCompactionTask::prepare_for_compaction(ClosureType& cl,

@@ -452,11 +468,12 @@
      ShenandoahHeapRegion* r = empty_regions.at(pos);
      r->set_new_top(r->bottom());
    }
  }
  
- void ShenandoahFullGC::calculate_target_humongous_objects() {
+ template<bool ALT_FWD>
+ void ShenandoahFullGC::calculate_target_humongous_objects_impl() {
    ShenandoahHeap* heap = ShenandoahHeap::heap();
  
    // Compute the new addresses for humongous objects. We need to do this after addresses
    // for regular objects are calculated, and we know what regions in heap suffix are
    // available for humongous moves.

@@ -488,11 +505,11 @@
        size_t start = to_end - num_regions;
  
        if (start >= to_begin && start != r->index()) {
          // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
          _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
-         old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom()));
+         SlidingForwarding::forward_to<ALT_FWD>(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
          to_end = start;
          continue;
        }
      }
  

@@ -500,10 +517,18 @@
      to_begin = r->index();
      to_end = r->index();
    }
  }
  
+ void ShenandoahFullGC::calculate_target_humongous_objects() {
+   if (UseAltGCForwarding) {
+     calculate_target_humongous_objects_impl<true>();
+   } else {
+     calculate_target_humongous_objects_impl<false>();
+   }
+ }
+ 
  class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
  private:
    ShenandoahHeap* const _heap;
  
  public:

@@ -737,10 +762,11 @@
      ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
      calculate_target_humongous_objects();
    }
  }
  
+ template <bool ALT_FWD>
  class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
  private:
    ShenandoahHeap* const _heap;
    ShenandoahMarkingContext* const _ctx;
  

@@ -748,12 +774,12 @@
    inline void do_oop_work(T* p) {
      T o = RawAccess<>::oop_load(p);
      if (!CompressedOops::is_null(o)) {
        oop obj = CompressedOops::decode_not_null(o);
        assert(_ctx->is_marked(obj), "must be marked");
-       if (obj->is_forwarded()) {
-         oop forw = obj->forwardee();
+       if (SlidingForwarding::is_forwarded(obj)) {
+         oop forw = SlidingForwarding::forwardee<ALT_FWD>(obj);
          RawAccess<IS_NOT_NULL>::oop_store(p, forw);
        }
      }
    }
  

@@ -766,14 +792,15 @@
    void do_oop(narrowOop* p) { do_oop_work(p); }
    void do_method(Method* m) {}
    void do_nmethod(nmethod* nm) {}
  };
  
+ template <bool ALT_FWD>
  class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
  private:
    ShenandoahHeap* const _heap;
-   ShenandoahAdjustPointersClosure _cl;
+   ShenandoahAdjustPointersClosure<ALT_FWD> _cl;
  
  public:
    ShenandoahAdjustPointersObjectClosure() :
      _heap(ShenandoahHeap::heap()) {
    }

@@ -792,39 +819,62 @@
    ShenandoahAdjustPointersTask() :
      WorkerTask("Shenandoah Adjust Pointers"),
      _heap(ShenandoahHeap::heap()) {
    }
  
-   void work(uint worker_id) {
+ private:
+   template <bool ALT_FWD>
+   void work_impl(uint worker_id) {
      ShenandoahParallelWorkerSession worker_session(worker_id);
-     ShenandoahAdjustPointersObjectClosure obj_cl;
+     ShenandoahAdjustPointersObjectClosure<ALT_FWD> obj_cl;
      ShenandoahHeapRegion* r = _regions.next();
      while (r != nullptr) {
        if (!r->is_humongous_continuation() && r->has_live()) {
          _heap->marked_object_iterate(r, &obj_cl);
        }
        r = _regions.next();
      }
    }
+ 
+ public:
+   void work(uint worker_id) {
+     if (UseAltGCForwarding) {
+       work_impl<true>(worker_id);
+     } else {
+       work_impl<false>(worker_id);
+     }
+   }
  };
  
  class ShenandoahAdjustRootPointersTask : public WorkerTask {
  private:
    ShenandoahRootAdjuster* _rp;
    PreservedMarksSet* _preserved_marks;
+ 
  public:
    ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
      WorkerTask("Shenandoah Adjust Root Pointers"),
      _rp(rp),
      _preserved_marks(preserved_marks) {}
  
-   void work(uint worker_id) {
+ private:
+   template <bool ALT_FWD>
+   void work_impl(uint worker_id) {
      ShenandoahParallelWorkerSession worker_session(worker_id);
-     ShenandoahAdjustPointersClosure cl;
+     ShenandoahAdjustPointersClosure<ALT_FWD> cl;
      _rp->roots_do(worker_id, &cl);
      _preserved_marks->get(worker_id)->adjust_during_full_gc();
    }
+ 
+ public:
+   void work(uint worker_id) {
+     if (UseAltGCForwarding) {
+       work_impl<true>(worker_id);
+     } else {
+       work_impl<false>(worker_id);
+     }
+   }
  };
  
  void ShenandoahFullGC::phase3_update_references() {
    GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
    ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);

@@ -847,10 +897,11 @@
  
    ShenandoahAdjustPointersTask adjust_pointers_task;
    workers->run_task(&adjust_pointers_task);
  }
  
+ template <bool ALT_FWD>
  class ShenandoahCompactObjectsClosure : public ObjectClosure {
  private:
    ShenandoahHeap* const _heap;
    uint            const _worker_id;
  

@@ -859,13 +910,13 @@
      _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
  
    void do_object(oop p) {
      assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
      size_t size = p->size();
-     if (p->is_forwarded()) {
+     if (SlidingForwarding::is_forwarded(p)) {
        HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
-       HeapWord* compact_to = cast_from_oop<HeapWord*>(p->forwardee());
+       HeapWord* compact_to = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(p));
        Copy::aligned_conjoint_words(compact_from, compact_to, size);
        oop new_obj = cast_to_oop(compact_to);
  
        ContinuationGCSupport::relativize_stack_chunk(new_obj);
        new_obj->init_mark();

@@ -883,25 +934,36 @@
      WorkerTask("Shenandoah Compact Objects"),
      _heap(ShenandoahHeap::heap()),
      _worker_slices(worker_slices) {
    }
  
-   void work(uint worker_id) {
+ private:
+   template <bool ALT_FWD>
+   void work_impl(uint worker_id) {
      ShenandoahParallelWorkerSession worker_session(worker_id);
      ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
  
-     ShenandoahCompactObjectsClosure cl(worker_id);
+     ShenandoahCompactObjectsClosure<ALT_FWD> cl(worker_id);
      ShenandoahHeapRegion* r = slice.next();
      while (r != nullptr) {
        assert(!r->is_humongous(), "must not get humongous regions here");
        if (r->has_live()) {
          _heap->marked_object_iterate(r, &cl);
        }
        r->set_top(r->new_top());
        r = slice.next();
      }
    }
+ 
+ public:
+   void work(uint worker_id) {
+     if (UseAltGCForwarding) {
+       work_impl<true>(worker_id);
+     } else {
+       work_impl<false>(worker_id);
+     }
+   }
  };
  
  class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
  private:
    ShenandoahHeap* const _heap;

@@ -953,11 +1015,12 @@
    size_t get_live() {
      return _live;
    }
  };
  
- void ShenandoahFullGC::compact_humongous_objects() {
+ template <bool ALT_FWD>
+ void ShenandoahFullGC::compact_humongous_objects_impl() {
    // Compact humongous regions, based on their fwdptr objects.
    //
    // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
    // humongous regions are already compacted, and do not require further moves, which alleviates
    // sliding costs. We may consider doing this in parallel in the future.

@@ -966,20 +1029,20 @@
  
    for (size_t c = heap->num_regions(); c > 0; c--) {
      ShenandoahHeapRegion* r = heap->get_region(c - 1);
      if (r->is_humongous_start()) {
        oop old_obj = cast_to_oop(r->bottom());
-       if (!old_obj->is_forwarded()) {
+       if (SlidingForwarding::is_not_forwarded(old_obj)) {
          // No need to move the object, it stays at the same slot
          continue;
        }
        size_t words_size = old_obj->size();
        size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
  
        size_t old_start = r->index();
        size_t old_end   = old_start + num_regions - 1;
-       size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
+       size_t new_start = heap->heap_region_index_containing(SlidingForwarding::forwardee<ALT_FWD>(old_obj));
        size_t new_end   = new_start + num_regions - 1;
        assert(old_start != new_start, "must be real move");
        assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
  
        Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);

@@ -1016,10 +1079,18 @@
        }
      }
    }
  }
  
+ void ShenandoahFullGC::compact_humongous_objects() {
+   if (UseAltGCForwarding) {
+     compact_humongous_objects_impl<true>();
+   } else {
+     compact_humongous_objects_impl<false>();
+   }
+ }
+ 
  // This is slightly different to ShHeap::reset_next_mark_bitmap:
  // we need to remain able to walk pinned regions.
  // Since pinned region do not move and don't get compacted, we will get holes with
  // unreachable objects in them (which may have pointers to unloaded Klasses and thus
  // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
< prev index next >