< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

Print this page
*** 1,7 ***
--- 1,8 ---
  /*
   * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   *
   * This code is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 only, as
   * published by the Free Software Foundation.

*** 28,24 ***
--- 29,27 ---
  #include "gc/shared/strongRootsScope.hpp"
  #include "gc/shared/taskTerminator.hpp"
  #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
  #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+ #include "gc/shenandoah/shenandoahGeneration.hpp"
  #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  #include "gc/shenandoah/shenandoahMark.inline.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahStringDedup.hpp"
  #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
+ #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  #include "memory/iterator.inline.hpp"
  #include "memory/resourceArea.hpp"
  #include "runtime/continuation.hpp"
  #include "runtime/threads.hpp"
  
+ template <ShenandoahGenerationType GENERATION>
  class ShenandoahConcurrentMarkingTask : public WorkerTask {
  private:
    ShenandoahConcurrentMark* const _cm;
    TaskTerminator* const           _terminator;
  

*** 54,27 ***
      WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
    }
  
    void work(uint worker_id) {
      ShenandoahHeap* heap = ShenandoahHeap::heap();
!     ShenandoahConcurrentWorkerSession worker_session(worker_id);
      ShenandoahSuspendibleThreadSetJoiner stsj;
!     ShenandoahReferenceProcessor* rp = heap->ref_processor();
      assert(rp != nullptr, "need reference processor");
      StringDedup::Requests requests;
!     _cm->mark_loop(worker_id, _terminator, rp,
-                    true /*cancellable*/,
                     ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
                     &requests);
    }
  };
  
  class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
  private:
    SATBMarkQueueSet& _satb_qset;
    OopClosure* const _cl;
-   uintx _claim_token;
  
  public:
    ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
      _satb_qset(satb_qset),
      _cl(cl)  {}
--- 58,30 ---
      WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
    }
  
    void work(uint worker_id) {
      ShenandoahHeap* heap = ShenandoahHeap::heap();
!     ShenandoahParallelWorkerSession worker_session(worker_id);
+     ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
      ShenandoahSuspendibleThreadSetJoiner stsj;
!     // Do not use active_generation() : we must use the gc_generation() set by
+     // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
+     // intervene to update active_generation, so we can't
+     // shenandoah_assert_generations_reconciled() here.
+     ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
      assert(rp != nullptr, "need reference processor");
      StringDedup::Requests requests;
!     _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
                     ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
                     &requests);
    }
  };
  
  class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
  private:
    SATBMarkQueueSet& _satb_qset;
    OopClosure* const _cl;
  
  public:
    ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
      _satb_qset(satb_qset),
      _cl(cl)  {}

*** 89,10 ***
--- 96,11 ---
        }
      }
    }
  };
  
+ template <ShenandoahGenerationType GENERATION>
  class ShenandoahFinalMarkingTask : public WorkerTask {
  private:
    ShenandoahConcurrentMark* _cm;
    TaskTerminator*           _terminator;
    bool                      _dedup_string;

*** 104,84 ***
  
    void work(uint worker_id) {
      ShenandoahHeap* heap = ShenandoahHeap::heap();
  
      ShenandoahParallelWorkerSession worker_session(worker_id);
-     ShenandoahReferenceProcessor* rp = heap->ref_processor();
      StringDedup::Requests requests;
  
      // First drain remaining SATB buffers.
      {
        ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
  
!       ShenandoahSATBBufferClosure cl(q);
        SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
        while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
        assert(!heap->has_forwarded_objects(), "Not expected");
  
!       ShenandoahMarkRefsClosure             mark_cl(q, rp);
        ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
                                                 ShenandoahIUBarrier ? &mark_cl : nullptr);
        Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
      }
!     _cm->mark_loop(worker_id, _terminator, rp,
-                    false /*not cancellable*/,
                     _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
                     &requests);
      assert(_cm->task_queues()->is_empty(), "Should be empty");
    }
  };
  
! ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
!   ShenandoahMark() {}
  
  // Mark concurrent roots during concurrent phases
  class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
  private:
    SuspendibleThreadSetJoiner          _sts_joiner;
    ShenandoahConcurrentRootScanner     _root_scanner;
    ShenandoahObjToScanQueueSet* const  _queue_set;
    ShenandoahReferenceProcessor* const _rp;
  
  public:
    ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
                                      ShenandoahReferenceProcessor* rp,
                                      ShenandoahPhaseTimings::Phase phase,
                                      uint nworkers);
    void work(uint worker_id);
  };
  
! ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
!                                                                      ShenandoahReferenceProcessor* rp,
!                                                                      ShenandoahPhaseTimings::Phase phase,
!                                                                      uint nworkers) :
    WorkerTask("Shenandoah Concurrent Mark Roots"),
    _root_scanner(nworkers, phase),
    _queue_set(qs),
    _rp(rp) {
    assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
  }
  
! void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
    ShenandoahConcurrentWorkerSession worker_session(worker_id);
    ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
!   ShenandoahMarkRefsClosure cl(q, _rp);
    _root_scanner.roots_do(&cl, worker_id);
  }
  
  void ShenandoahConcurrentMark::mark_concurrent_roots() {
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    assert(!heap->has_forwarded_objects(), "Not expected");
  
-   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
- 
    WorkerThreads* workers = heap->workers();
!   ShenandoahReferenceProcessor* rp = heap->ref_processor();
!   task_queues()->reserve(workers->active_workers());
!   ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
! 
!   workers->run_task(&task);
  }
  
  class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
  private:
    SATBMarkQueueSet& _qset;
--- 112,118 ---
  
    void work(uint worker_id) {
      ShenandoahHeap* heap = ShenandoahHeap::heap();
  
      ShenandoahParallelWorkerSession worker_session(worker_id);
      StringDedup::Requests requests;
+     ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
+     shenandoah_assert_generations_reconciled();
  
      // First drain remaining SATB buffers.
      {
        ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
+       ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
  
!       ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
        SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
        while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
        assert(!heap->has_forwarded_objects(), "Not expected");
  
!       ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q);
        ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
                                                 ShenandoahIUBarrier ? &mark_cl : nullptr);
        Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
      }
!     _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
                     _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
                     &requests);
      assert(_cm->task_queues()->is_empty(), "Should be empty");
    }
  };
  
! ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
!   ShenandoahMark(generation) {}
  
  // Mark concurrent roots during concurrent phases
+ template <ShenandoahGenerationType GENERATION>
  class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
  private:
    SuspendibleThreadSetJoiner          _sts_joiner;
    ShenandoahConcurrentRootScanner     _root_scanner;
    ShenandoahObjToScanQueueSet* const  _queue_set;
+   ShenandoahObjToScanQueueSet* const  _old_queue_set;
    ShenandoahReferenceProcessor* const _rp;
  
  public:
    ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
+                                     ShenandoahObjToScanQueueSet* old,
                                      ShenandoahReferenceProcessor* rp,
                                      ShenandoahPhaseTimings::Phase phase,
                                      uint nworkers);
    void work(uint worker_id);
  };
  
! template <ShenandoahGenerationType GENERATION>
! ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
!                                                                                  ShenandoahObjToScanQueueSet* old,
!                                                                                  ShenandoahReferenceProcessor* rp,
+                                                                                  ShenandoahPhaseTimings::Phase phase,
+                                                                                  uint nworkers) :
    WorkerTask("Shenandoah Concurrent Mark Roots"),
    _root_scanner(nworkers, phase),
    _queue_set(qs),
+   _old_queue_set(old),
    _rp(rp) {
    assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
  }
  
! template <ShenandoahGenerationType GENERATION>
+ void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
    ShenandoahConcurrentWorkerSession worker_session(worker_id);
    ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
!   ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
+           nullptr : _old_queue_set->queue(worker_id);
+   ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
    _root_scanner.roots_do(&cl, worker_id);
  }
  
  void ShenandoahConcurrentMark::mark_concurrent_roots() {
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    assert(!heap->has_forwarded_objects(), "Not expected");
  
    WorkerThreads* workers = heap->workers();
!   ShenandoahReferenceProcessor* rp = _generation->ref_processor();
!   _generation->reserve_task_queues(workers->active_workers());
!   switch (_generation->type()) {
!     case YOUNG: {
!       ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
+                                                     ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+       workers->run_task(&task);
+       break;
+     }
+     case GLOBAL: {
+       assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
+       ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), nullptr, rp,
+                                                      ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+       workers->run_task(&task);
+       break;
+     }
+     case NON_GEN: {
+       assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
+       ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), nullptr, rp,
+                                                       ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+       workers->run_task(&task);
+       break;
+     }
+     case OLD: {
+       // We use a YOUNG generation cycle to bootstrap concurrent old marking.
+       ShouldNotReachHere();
+       break;
+     }
+     default:
+       ShouldNotReachHere();
+   }
  }
  
  class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
  private:
    SATBMarkQueueSet& _qset;

*** 199,16 ***
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    WorkerThreads* workers = heap->workers();
    uint nworkers = workers->active_workers();
    task_queues()->reserve(nworkers);
  
    ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
    ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
    for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
!     TaskTerminator terminator(nworkers, task_queues());
!     ShenandoahConcurrentMarkingTask task(this, &terminator);
!     workers->run_task(&task);
  
      if (heap->cancelled_gc()) {
        // GC is cancelled, break out.
        break;
      }
--- 241,42 ---
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    WorkerThreads* workers = heap->workers();
    uint nworkers = workers->active_workers();
    task_queues()->reserve(nworkers);
  
+   ShenandoahGenerationType gen_type = _generation->type();
    ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
    ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
    for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
!     switch (gen_type) {
!       case YOUNG: {
!         TaskTerminator terminator(nworkers, task_queues());
+         ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
+         workers->run_task(&task);
+         break;
+       }
+       case OLD: {
+         TaskTerminator terminator(nworkers, task_queues());
+         ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
+         workers->run_task(&task);
+         break;
+       }
+       case GLOBAL: {
+         TaskTerminator terminator(nworkers, task_queues());
+         ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator);
+         workers->run_task(&task);
+         break;
+       }
+       case NON_GEN: {
+         TaskTerminator terminator(nworkers, task_queues());
+         ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator);
+         workers->run_task(&task);
+         break;
+       }
+       default:
+         ShouldNotReachHere();
+     }
  
      if (heap->cancelled_gc()) {
        // GC is cancelled, break out.
        break;
      }

*** 228,16 ***
  void ShenandoahConcurrentMark::finish_mark() {
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
    assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
    finish_mark_work();
    assert(task_queues()->is_empty(), "Should be empty");
!   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
-   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
  
!   ShenandoahHeap* const heap = ShenandoahHeap::heap();
!   heap->set_concurrent_mark_in_progress(false);
-   heap->mark_complete_marking_context();
  
    end_mark();
  }
  
  void ShenandoahConcurrentMark::finish_mark_work() {
--- 296,14 ---
  void ShenandoahConcurrentMark::finish_mark() {
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
    assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
    finish_mark_work();
    assert(task_queues()->is_empty(), "Should be empty");
!   TASKQUEUE_STATS_ONLY(task_queues()->print_and_reset_taskqueue_stats(""));
  
!   _generation->set_concurrent_mark_in_progress(false);
!   _generation->set_mark_complete();
  
    end_mark();
  }
  
  void ShenandoahConcurrentMark::finish_mark_work() {

*** 253,17 ***
    uint nworkers = heap->workers()->active_workers();
    task_queues()->reserve(nworkers);
  
    StrongRootsScope scope(nworkers);
    TaskTerminator terminator(nworkers, task_queues());
-   ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
-   heap->workers()->run_task(&task);
  
!   assert(task_queues()->is_empty(), "Should be empty");
! }
  
  
! void ShenandoahConcurrentMark::cancel() {
-   clear();
-   ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
-   rp->abandon_partial_discovery();
  }
--- 319,34 ---
    uint nworkers = heap->workers()->active_workers();
    task_queues()->reserve(nworkers);
  
    StrongRootsScope scope(nworkers);
    TaskTerminator terminator(nworkers, task_queues());
  
!   switch (_generation->type()) {
!     case YOUNG:{
+       ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+       heap->workers()->run_task(&task);
+       break;
+     }
+     case OLD:{
+       ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+       heap->workers()->run_task(&task);
+       break;
+     }
+     case GLOBAL:{
+       ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+       heap->workers()->run_task(&task);
+       break;
+     }
+     case NON_GEN:{
+       ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
+       heap->workers()->run_task(&task);
+       break;
+     }
+     default:
+       ShouldNotReachHere();
+   }
  
  
!   assert(task_queues()->is_empty(), "Should be empty");
  }
< prev index next >