< prev index next > src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
Print this page
/*
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+ #include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahStringDedup.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
+ #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/continuation.hpp"
#include "runtime/threads.hpp"
+ template <ShenandoahGenerationType GENERATION>
class ShenandoahConcurrentMarkingTask : public WorkerTask {
private:
ShenandoahConcurrentMark* const _cm;
TaskTerminator* const _terminator;
WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
}
void work(uint worker_id) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
! ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahSuspendibleThreadSetJoiner stsj;
! ShenandoahReferenceProcessor* rp = heap->ref_processor();
assert(rp != nullptr, "need reference processor");
StringDedup::Requests requests;
! _cm->mark_loop(worker_id, _terminator, rp,
- true /*cancellable*/,
ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
}
};
WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
}
void work(uint worker_id) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
! ShenandoahParallelWorkerSession worker_session(worker_id);
+ ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
ShenandoahSuspendibleThreadSetJoiner stsj;
! // Do not use active_generation() : we must use the gc_generation() set by
+ // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
+ // intervene to update active_generation, so we can't
+ // shenandoah_assert_generations_reconciled() here.
+ ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
assert(rp != nullptr, "need reference processor");
StringDedup::Requests requests;
! _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
}
};
}
}
}
};
+ template <ShenandoahGenerationType GENERATION>
class ShenandoahFinalMarkingTask : public WorkerTask {
private:
ShenandoahConcurrentMark* _cm;
TaskTerminator* _terminator;
bool _dedup_string;
void work(uint worker_id) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
StringDedup::Requests requests;
// First drain remaining SATB buffers.
{
ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
! ShenandoahSATBBufferClosure cl(q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
assert(!heap->has_forwarded_objects(), "Not expected");
! ShenandoahMarkRefsClosure mark_cl(q, rp);
ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
ShenandoahIUBarrier ? &mark_cl : nullptr);
Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
}
! _cm->mark_loop(worker_id, _terminator, rp,
- false /*not cancellable*/,
_dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
assert(_cm->task_queues()->is_empty(), "Should be empty");
}
};
! ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
! ShenandoahMark() {}
// Mark concurrent roots during concurrent phases
class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
private:
SuspendibleThreadSetJoiner _sts_joiner;
ShenandoahConcurrentRootScanner _root_scanner;
ShenandoahObjToScanQueueSet* const _queue_set;
ShenandoahReferenceProcessor* const _rp;
public:
ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers);
void work(uint worker_id);
};
! ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
! ShenandoahReferenceProcessor* rp,
! ShenandoahPhaseTimings::Phase phase,
! uint nworkers) :
WorkerTask("Shenandoah Concurrent Mark Roots"),
_root_scanner(nworkers, phase),
_queue_set(qs),
_rp(rp) {
assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
}
! void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
! ShenandoahMarkRefsClosure cl(q, _rp);
_root_scanner.roots_do(&cl, worker_id);
}
void ShenandoahConcurrentMark::mark_concurrent_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
assert(!heap->has_forwarded_objects(), "Not expected");
- TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
-
WorkerThreads* workers = heap->workers();
! ShenandoahReferenceProcessor* rp = heap->ref_processor();
! task_queues()->reserve(workers->active_workers());
! ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
!
! workers->run_task(&task);
}
class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
private:
SATBMarkQueueSet& _qset;
void work(uint worker_id) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahParallelWorkerSession worker_session(worker_id);
StringDedup::Requests requests;
+ ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
+ shenandoah_assert_generations_reconciled();
// First drain remaining SATB buffers.
{
ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
+ ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
! ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
assert(!heap->has_forwarded_objects(), "Not expected");
! ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q);
ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
ShenandoahIUBarrier ? &mark_cl : nullptr);
Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
}
! _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
_dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
assert(_cm->task_queues()->is_empty(), "Should be empty");
}
};
! ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
! ShenandoahMark(generation) {}
// Mark concurrent roots during concurrent phases
+ template <ShenandoahGenerationType GENERATION>
class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
private:
SuspendibleThreadSetJoiner _sts_joiner;
ShenandoahConcurrentRootScanner _root_scanner;
ShenandoahObjToScanQueueSet* const _queue_set;
+ ShenandoahObjToScanQueueSet* const _old_queue_set;
ShenandoahReferenceProcessor* const _rp;
public:
ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
+ ShenandoahObjToScanQueueSet* old,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers);
void work(uint worker_id);
};
! template <ShenandoahGenerationType GENERATION>
! ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
! ShenandoahObjToScanQueueSet* old,
! ShenandoahReferenceProcessor* rp,
+ ShenandoahPhaseTimings::Phase phase,
+ uint nworkers) :
WorkerTask("Shenandoah Concurrent Mark Roots"),
_root_scanner(nworkers, phase),
_queue_set(qs),
+ _old_queue_set(old),
_rp(rp) {
assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
}
! template <ShenandoahGenerationType GENERATION>
+ void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
! ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
+ nullptr : _old_queue_set->queue(worker_id);
+ ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
_root_scanner.roots_do(&cl, worker_id);
}
void ShenandoahConcurrentMark::mark_concurrent_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
assert(!heap->has_forwarded_objects(), "Not expected");
WorkerThreads* workers = heap->workers();
! ShenandoahReferenceProcessor* rp = _generation->ref_processor();
! _generation->reserve_task_queues(workers->active_workers());
! switch (_generation->type()) {
! case YOUNG: {
! ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case GLOBAL: {
+ assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
+ ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), nullptr, rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case NON_GEN: {
+ assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
+ ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), nullptr, rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case OLD: {
+ // We use a YOUNG generation cycle to bootstrap concurrent old marking.
+ ShouldNotReachHere();
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
}
class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
private:
SATBMarkQueueSet& _qset;
ShenandoahHeap* const heap = ShenandoahHeap::heap();
WorkerThreads* workers = heap->workers();
uint nworkers = workers->active_workers();
task_queues()->reserve(nworkers);
ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
! TaskTerminator terminator(nworkers, task_queues());
! ShenandoahConcurrentMarkingTask task(this, &terminator);
! workers->run_task(&task);
if (heap->cancelled_gc()) {
// GC is cancelled, break out.
break;
}
size_t before = qset.completed_buffers_num();
! Handshake::execute(&flush_satb);
size_t after = qset.completed_buffers_num();
if (before == after) {
// No more retries needed, break out.
break;
ShenandoahHeap* const heap = ShenandoahHeap::heap();
WorkerThreads* workers = heap->workers();
uint nworkers = workers->active_workers();
task_queues()->reserve(nworkers);
+ ShenandoahGenerationType gen_type = _generation->type();
ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
! switch (gen_type) {
! case YOUNG: {
! TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case OLD: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case GLOBAL: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case NON_GEN: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
if (heap->cancelled_gc()) {
// GC is cancelled, break out.
break;
}
size_t before = qset.completed_buffers_num();
! {
+ ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_mark_satb_flush, true);
+ Handshake::execute(&flush_satb);
+ }
size_t after = qset.completed_buffers_num();
if (before == after) {
// No more retries needed, break out.
break;
void ShenandoahConcurrentMark::finish_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
finish_mark_work();
assert(task_queues()->is_empty(), "Should be empty");
! TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
- TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
! ShenandoahHeap* const heap = ShenandoahHeap::heap();
! heap->set_concurrent_mark_in_progress(false);
- heap->mark_complete_marking_context();
end_mark();
}
void ShenandoahConcurrentMark::finish_mark_work() {
void ShenandoahConcurrentMark::finish_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
finish_mark_work();
assert(task_queues()->is_empty(), "Should be empty");
! TASKQUEUE_STATS_ONLY(task_queues()->print_and_reset_taskqueue_stats(""));
! _generation->set_concurrent_mark_in_progress(false);
! _generation->set_mark_complete();
end_mark();
}
void ShenandoahConcurrentMark::finish_mark_work() {
uint nworkers = heap->workers()->active_workers();
task_queues()->reserve(nworkers);
StrongRootsScope scope(nworkers);
TaskTerminator terminator(nworkers, task_queues());
- ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
- heap->workers()->run_task(&task);
! assert(task_queues()->is_empty(), "Should be empty");
! }
! void ShenandoahConcurrentMark::cancel() {
- clear();
- ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
- rp->abandon_partial_discovery();
}
uint nworkers = heap->workers()->active_workers();
task_queues()->reserve(nworkers);
StrongRootsScope scope(nworkers);
TaskTerminator terminator(nworkers, task_queues());
! switch (_generation->type()) {
! case YOUNG:{
+ ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case OLD:{
+ ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case GLOBAL:{
+ ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case NON_GEN:{
+ ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
! assert(task_queues()->is_empty(), "Should be empty");
}
< prev index next >