< prev index next > src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
Print this page
#include "compiler/oopMap.hpp"
#include "gc/shared/continuationGCSupport.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
+ #include "gc/shared/slidingForwarding.inline.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
{
// The rest of code performs region moves, where region status is undefined
// until all phases run together.
ShenandoahHeapLocker lock(heap->lock());
+ SlidingForwarding::begin();
+
phase2_calculate_target_addresses(worker_slices);
OrderAccess::fence();
phase3_update_references();
{
// Epilogue
_preserved_marks->restore(heap->workers());
_preserved_marks->reclaim();
+ SlidingForwarding::end();
}
// Resize metaspace
MetaspaceGC::compute_new_size();
ShenandoahSTWMark mark(true /*full_gc*/);
mark.mark();
heap->parallel_cleaning(true /* full_gc */);
}
+ template <bool ALT_FWD>
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
PreservedMarks* const _preserved_marks;
ShenandoahHeap* const _heap;
GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
// Object fits into current region, record new location, if object does not move:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(nullptr, p);
if (_compact_point != cast_from_oop<HeapWord*>(p)) {
_preserved_marks->push_if_necessary(p, p->mark());
! p->forward_to(cast_to_oop(_compact_point));
}
_compact_point += obj_size;
}
};
// Object fits into current region, record new location, if object does not move:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(nullptr, p);
if (_compact_point != cast_from_oop<HeapWord*>(p)) {
_preserved_marks->push_if_necessary(p, p->mark());
! SlidingForwarding::forward_to<ALT_FWD>(p, cast_to_oop(_compact_point));
}
_compact_point += obj_size;
}
};
// moves are special cased here, because their moves are handled separately.
return r->is_stw_move_allowed() && !r->is_humongous();
}
void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
+
+ private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
ShenandoahHeapRegionSetIterator it(slice);
ShenandoahHeapRegion* from_region = it.next();
// No work?
// Remember empty regions and reuse them as needed.
ResourceMark rm;
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
! ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
while (from_region != nullptr) {
assert(is_candidate_region(from_region), "Sanity");
cl.set_from_region(from_region);
// Remember empty regions and reuse them as needed.
ResourceMark rm;
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
! ShenandoahPrepareForCompactionObjectClosure<ALT_FWD> cl(_preserved_marks->get(worker_id), empty_regions, from_region);
while (from_region != nullptr) {
assert(is_candidate_region(from_region), "Sanity");
cl.set_from_region(from_region);
r->set_new_top(r->bottom());
}
}
};
! void ShenandoahFullGC::calculate_target_humongous_objects() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
// Compute the new addresses for humongous objects. We need to do this after addresses
// for regular objects are calculated, and we know what regions in heap suffix are
// available for humongous moves.
r->set_new_top(r->bottom());
}
}
};
! template <bool ALT_FWD>
+ void ShenandoahFullGC::calculate_target_humongous_objects_impl() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
// Compute the new addresses for humongous objects. We need to do this after addresses
// for regular objects are calculated, and we know what regions in heap suffix are
// available for humongous moves.
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->index()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
_preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
! old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom()));
to_end = start;
continue;
}
}
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->index()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
_preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
! SlidingForwarding::forward_to<ALT_FWD>(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
to_end = start;
continue;
}
}
to_begin = r->index();
to_end = r->index();
}
}
+ void ShenandoahFullGC::calculate_target_humongous_objects() {
+ if (UseAltGCForwarding) {
+ calculate_target_humongous_objects_impl<true>();
+ } else {
+ calculate_target_humongous_objects_impl<false>();
+ }
+ }
+
class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
private:
ShenandoahHeap* const _heap;
public:
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
calculate_target_humongous_objects();
}
}
+ template <bool ALT_FWD>
class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
private:
ShenandoahHeap* const _heap;
ShenandoahMarkingContext* const _ctx;
inline void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! if (obj->is_forwarded()) {
! oop forw = obj->forwardee();
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
}
inline void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! if (SlidingForwarding::is_forwarded(obj)) {
! oop forw = SlidingForwarding::forwardee<ALT_FWD>(obj);
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
}
void do_oop(narrowOop* p) { do_oop_work(p); }
void do_method(Method* m) {}
void do_nmethod(nmethod* nm) {}
};
class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
! ShenandoahAdjustPointersClosure _cl;
public:
ShenandoahAdjustPointersObjectClosure() :
_heap(ShenandoahHeap::heap()) {
}
void do_oop(narrowOop* p) { do_oop_work(p); }
void do_method(Method* m) {}
void do_nmethod(nmethod* nm) {}
};
+ template <bool ALT_FWD>
class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
! ShenandoahAdjustPointersClosure<ALT_FWD> _cl;
public:
ShenandoahAdjustPointersObjectClosure() :
_heap(ShenandoahHeap::heap()) {
}
ShenandoahAdjustPointersTask() :
WorkerTask("Shenandoah Adjust Pointers"),
_heap(ShenandoahHeap::heap()) {
}
! void work(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
! ShenandoahAdjustPointersObjectClosure obj_cl;
ShenandoahHeapRegion* r = _regions.next();
while (r != nullptr) {
if (!r->is_humongous_continuation() && r->has_live()) {
_heap->marked_object_iterate(r, &obj_cl);
}
r = _regions.next();
}
}
};
class ShenandoahAdjustRootPointersTask : public WorkerTask {
private:
ShenandoahRootAdjuster* _rp;
PreservedMarksSet* _preserved_marks;
public:
ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
WorkerTask("Shenandoah Adjust Root Pointers"),
_rp(rp),
_preserved_marks(preserved_marks) {}
! void work(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
! ShenandoahAdjustPointersClosure cl;
_rp->roots_do(worker_id, &cl);
_preserved_marks->get(worker_id)->adjust_during_full_gc();
}
};
void ShenandoahFullGC::phase3_update_references() {
GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
ShenandoahAdjustPointersTask() :
WorkerTask("Shenandoah Adjust Pointers"),
_heap(ShenandoahHeap::heap()) {
}
! private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
! ShenandoahAdjustPointersObjectClosure<ALT_FWD> obj_cl;
ShenandoahHeapRegion* r = _regions.next();
while (r != nullptr) {
if (!r->is_humongous_continuation() && r->has_live()) {
_heap->marked_object_iterate(r, &obj_cl);
}
r = _regions.next();
}
}
+
+ public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
class ShenandoahAdjustRootPointersTask : public WorkerTask {
private:
ShenandoahRootAdjuster* _rp;
PreservedMarksSet* _preserved_marks;
+
public:
ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
WorkerTask("Shenandoah Adjust Root Pointers"),
_rp(rp),
_preserved_marks(preserved_marks) {}
! private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
! ShenandoahAdjustPointersClosure<ALT_FWD> cl;
_rp->roots_do(worker_id, &cl);
_preserved_marks->get(worker_id)->adjust_during_full_gc();
}
+
+ public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
void ShenandoahFullGC::phase3_update_references() {
GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
ShenandoahAdjustPointersTask adjust_pointers_task;
workers->run_task(&adjust_pointers_task);
}
+ template <bool ALT_FWD>
class ShenandoahCompactObjectsClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
uint const _worker_id;
_heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = p->size();
! if (p->is_forwarded()) {
HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
! HeapWord* compact_to = cast_from_oop<HeapWord*>(p->forwardee());
assert(compact_from != compact_to, "Forwarded object should move");
Copy::aligned_conjoint_words(compact_from, compact_to, size);
oop new_obj = cast_to_oop(compact_to);
ContinuationGCSupport::relativize_stack_chunk(new_obj);
_heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = p->size();
! if (SlidingForwarding::is_forwarded(p)) {
HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
! HeapWord* compact_to = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(p));
assert(compact_from != compact_to, "Forwarded object should move");
Copy::aligned_conjoint_words(compact_from, compact_to, size);
oop new_obj = cast_to_oop(compact_to);
ContinuationGCSupport::relativize_stack_chunk(new_obj);
WorkerTask("Shenandoah Compact Objects"),
_heap(ShenandoahHeap::heap()),
_worker_slices(worker_slices) {
}
! void work(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
! ShenandoahCompactObjectsClosure cl(worker_id);
ShenandoahHeapRegion* r = slice.next();
while (r != nullptr) {
assert(!r->is_humongous(), "must not get humongous regions here");
if (r->has_live()) {
_heap->marked_object_iterate(r, &cl);
}
r->set_top(r->new_top());
r = slice.next();
}
}
};
class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
private:
ShenandoahHeap* const _heap;
WorkerTask("Shenandoah Compact Objects"),
_heap(ShenandoahHeap::heap()),
_worker_slices(worker_slices) {
}
! private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
! ShenandoahCompactObjectsClosure<ALT_FWD> cl(worker_id);
ShenandoahHeapRegion* r = slice.next();
while (r != nullptr) {
assert(!r->is_humongous(), "must not get humongous regions here");
if (r->has_live()) {
_heap->marked_object_iterate(r, &cl);
}
r->set_top(r->new_top());
r = slice.next();
}
}
+
+ public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
private:
ShenandoahHeap* const _heap;
size_t get_live() {
return _live;
}
};
! void ShenandoahFullGC::compact_humongous_objects() {
// Compact humongous regions, based on their fwdptr objects.
//
// This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
// humongous regions are already compacted, and do not require further moves, which alleviates
// sliding costs. We may consider doing this in parallel in future.
size_t get_live() {
return _live;
}
};
! template <bool ALT_FWD>
+ void ShenandoahFullGC::compact_humongous_objects_impl() {
// Compact humongous regions, based on their fwdptr objects.
//
// This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
// humongous regions are already compacted, and do not require further moves, which alleviates
// sliding costs. We may consider doing this in parallel in future.
for (size_t c = heap->num_regions(); c > 0; c--) {
ShenandoahHeapRegion* r = heap->get_region(c - 1);
if (r->is_humongous_start()) {
oop old_obj = cast_to_oop(r->bottom());
! if (!old_obj->is_forwarded()) {
// No need to move the object, it stays at the same slot
continue;
}
size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->index();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
size_t new_end = new_start + num_regions - 1;
assert(old_start != new_start, "must be real move");
assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
for (size_t c = heap->num_regions(); c > 0; c--) {
ShenandoahHeapRegion* r = heap->get_region(c - 1);
if (r->is_humongous_start()) {
oop old_obj = cast_to_oop(r->bottom());
! if (SlidingForwarding::is_not_forwarded(old_obj)) {
// No need to move the object, it stays at the same slot
continue;
}
size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->index();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(SlidingForwarding::forwardee<ALT_FWD>(old_obj));
size_t new_end = new_start + num_regions - 1;
assert(old_start != new_start, "must be real move");
assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
}
}
}
}
+ void ShenandoahFullGC::compact_humongous_objects() {
+ if (UseAltGCForwarding) {
+ compact_humongous_objects_impl<true>();
+ } else {
+ compact_humongous_objects_impl<false>();
+ }
+ }
+
// This is slightly different to ShHeap::reset_next_mark_bitmap:
// we need to remain able to walk pinned regions.
// Since pinned region do not move and don't get compacted, we will get holes with
// unreachable objects in them (which may have pointers to unloaded Klasses and thus
// cannot be iterated over using oop->size(). The only way to safely iterate over those is using
< prev index next >