< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp

Print this page

        

@@ -28,16 +28,14 @@
 #include "gc/shared/referenceProcessor.hpp"
 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
 #include "gc/shared/workgroup.hpp"
 #include "gc/shared/weakProcessor.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
-#include "gc/shenandoah/shenandoahForwarding.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"

@@ -51,11 +49,10 @@
 #include "gc/shenandoah/shenandoahVerifier.hpp"
 
 #include "memory/iterator.hpp"
 #include "memory/metaspace.hpp"
 #include "memory/resourceArea.hpp"
-#include "memory/universe.hpp"
 
 /**
  * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
  * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
  * is incremental-update-based.

@@ -159,20 +156,19 @@
   }
 };
 
 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
 private:
-  ShenandoahCSetRootScanner* _rp;
+  ShenandoahRootProcessor* _rp;
   ShenandoahHeap* _heap;
   ShenandoahCsetCodeRootsIterator* _cset_coderoots;
-  ShenandoahStringDedupRoots       _dedup_roots;
-
 public:
-  ShenandoahInitTraversalCollectionTask(ShenandoahCSetRootScanner* rp) :
+  ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahCsetCodeRootsIterator* cset_coderoots) :
     AbstractGangTask("Shenandoah Init Traversal Collection"),
     _rp(rp),
-    _heap(ShenandoahHeap::heap()) {}
+    _heap(ShenandoahHeap::heap()),
+    _cset_coderoots(cset_coderoots) {}
 
   void work(uint worker_id) {
     ShenandoahParallelWorkerSession worker_session(worker_id);
 
     ShenandoahEvacOOMScope oom_evac_scope;

@@ -190,17 +186,18 @@
     {
       ShenandoahTraversalClosure roots_cl(q, rp);
       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
       if (unload_classes) {
-        _rp->roots_do(worker_id, &roots_cl, NULL, &code_cl);
+        _rp->process_strong_roots(&roots_cl, &cld_cl, NULL, NULL, worker_id);
+        // Need to pre-evac code roots here. Otherwise we might see from-space constants.
+        ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times();
+        ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
+        _cset_coderoots->possibly_parallel_blobs_do(&code_cl);
       } else {
-        _rp->roots_do(worker_id, &roots_cl, &cld_cl, &code_cl);
+        _rp->process_all_roots(&roots_cl, &cld_cl, &code_cl, NULL, worker_id);
       }
-
-      AlwaysTrueClosure is_alive;
-      _dedup_roots.oops_do(&is_alive, &roots_cl, worker_id);
     }
   }
 };
 
 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {

@@ -224,15 +221,15 @@
   }
 };
 
 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 private:
-  ShenandoahAllRootScanner* _rp;
+  ShenandoahRootProcessor* _rp;
   ShenandoahTaskTerminator* _terminator;
   ShenandoahHeap* _heap;
 public:
-  ShenandoahFinalTraversalCollectionTask(ShenandoahAllRootScanner* rp, ShenandoahTaskTerminator* terminator) :
+  ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahTaskTerminator* terminator) :
     AbstractGangTask("Shenandoah Final Traversal Collection"),
     _rp(rp),
     _terminator(terminator),
     _heap(ShenandoahHeap::heap()) {}
 

@@ -267,27 +264,27 @@
     // and the references to the oops are updated during init pause. New nmethods are handled
     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
     // roots here.
     if (!_heap->is_degenerated_gc_in_progress()) {
       ShenandoahTraversalClosure roots_cl(q, rp);
+      CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
       if (unload_classes) {
         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
-        _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
+        _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
       } else {
-        CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
-        _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
+        _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
       }
     } else {
       ShenandoahTraversalDegenClosure roots_cl(q, rp);
+      CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
       if (unload_classes) {
         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
-        _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
+        _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
       } else {
-        CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
-        _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
+        _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
       }
     }
 
     {
       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();

@@ -303,13 +300,10 @@
 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
   _heap(heap),
   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
   _traversal_set(ShenandoahHeapRegionSet()) {
 
-  // Traversal does not support concurrent code root scanning
-  FLAG_SET_DEFAULT(ShenandoahConcurrentScanCodeRoots, false);
-
   uint num_queues = heap->max_workers();
   for (uint i = 0; i < num_queues; ++i) {
     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
     task_queue->initialize();
     _task_queues->register_queue(i, task_queue);

@@ -408,12 +402,15 @@
 #endif
 
     {
       uint nworkers = _heap->workers()->active_workers();
       task_queues()->reserve(nworkers);
-      ShenandoahCSetRootScanner rp(nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
-      ShenandoahInitTraversalCollectionTask traversal_task(&rp);
+      ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
+
+      ShenandoahCsetCodeRootsIterator cset_coderoots = ShenandoahCodeRoots::cset_iterator();
+
+      ShenandoahInitTraversalCollectionTask traversal_task(&rp, &cset_coderoots);
       _heap->workers()->run_task(&traversal_task);
     }
 
 #if defined(COMPILER2) || INCLUDE_JVMCI
     DerivedPointerTable::update_pointers();

@@ -550,10 +547,12 @@
   }
   return false;
 }
 
 void ShenandoahTraversalGC::concurrent_traversal_collection() {
+  ClassLoaderDataGraph::clear_claimed_marks();
+
   ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
   if (!_heap->cancelled_gc()) {
     uint nworkers = _heap->workers()->active_workers();
     task_queues()->reserve(nworkers);
     ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination);

@@ -578,11 +577,11 @@
     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
     uint nworkers = _heap->workers()->active_workers();
     task_queues()->reserve(nworkers);
 
     // Finish traversal
-    ShenandoahAllRootScanner rp(nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
+    ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
 
     ShenandoahTaskTerminator terminator(nworkers, task_queues());
     ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
     _heap->workers()->run_task(&task);

@@ -594,14 +593,15 @@
   if (!_heap->cancelled_gc() && _heap->process_references()) {
     weak_refs_work();
   }
 
   if (!_heap->cancelled_gc()) {
-    fixup_roots();
     if (_heap->unload_classes()) {
       _heap->unload_classes_and_cleanup_tables(false);
     }
+
+    fixup_roots();
   }
 
   if (!_heap->cancelled_gc()) {
     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
     TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());

@@ -631,11 +631,11 @@
         bool not_allocated = ctx->top_at_mark_start(r) == r->top();
 
         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
         if (r->is_humongous_start() && candidate) {
           // Trash humongous.
-          HeapWord* humongous_obj = r->bottom() + ShenandoahForwarding::word_size();
+          HeapWord* humongous_obj = r->bottom() + ShenandoahBrooksPointer::word_size();
           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
           r->make_trash_immediate();
           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
             i++;
             r = _heap->get_region(i);

@@ -687,32 +687,33 @@
   inline void do_oop(narrowOop* p) { do_oop_work(p); }
 };
 
 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
 private:
-  ShenandoahRootUpdater* _rp;
+  ShenandoahRootProcessor* _rp;
 
 public:
-  ShenandoahTraversalFixRootsTask(ShenandoahRootUpdater* rp) :
+  ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
     AbstractGangTask("Shenandoah traversal fix roots"),
     _rp(rp) {
     assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
   }
 
   void work(uint worker_id) {
     ShenandoahParallelWorkerSession worker_session(worker_id);
     ShenandoahTraversalFixRootsClosure cl;
-    ShenandoahForwardedIsAliveClosure is_alive;
-    _rp->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahTraversalFixRootsClosure>(worker_id, &is_alive, &cl);
+    MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
+    CLDToOopClosure cldCl(&cl, ClassLoaderData::_claim_strong);
+    _rp->update_all_roots<ShenandoahForwardedIsAliveClosure>(&cl, &cldCl, &blobsCl, NULL, worker_id);
   }
 };
 
 void ShenandoahTraversalGC::fixup_roots() {
 #if defined(COMPILER2) || INCLUDE_JVMCI
   DerivedPointerTable::clear();
 #endif
-  ShenandoahRootUpdater rp(_heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots, true /* update code cache */);
+  ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
   ShenandoahTraversalFixRootsTask update_roots_task(&rp);
   _heap->workers()->run_task(&update_roots_task);
 #if defined(COMPILER2) || INCLUDE_JVMCI
   DerivedPointerTable::update_pointers();
 #endif

@@ -766,10 +767,33 @@
 
   void do_oop(narrowOop* p) { do_oop_work(p); }
   void do_oop(oop* p)       { do_oop_work(p); }
 };
 
+class ShenandoahTraversalWeakUpdateClosure : public OopClosure {
+private:
+  template <class T>
+  inline void do_oop_work(T* p) {
+    // Cannot call maybe_update_with_forwarded, because on traversal-degen
+    // path the collection set is already dropped. Instead, do the unguarded store.
+    // TODO: This can be fixed after degen-traversal stops dropping cset.
+    T o = RawAccess<>::oop_load(p);
+    if (!CompressedOops::is_null(o)) {
+      oop obj = CompressedOops::decode_not_null(o);
+      obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
+      shenandoah_assert_marked(p, obj);
+      RawAccess<IS_NOT_NULL>::oop_store(p, obj);
+    }
+  }
+
+public:
+  ShenandoahTraversalWeakUpdateClosure() {}
+
+  void do_oop(narrowOop* p) { do_oop_work(p); }
+  void do_oop(oop* p)       { do_oop_work(p); }
+};
+
 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
 private:
   ShenandoahObjToScanQueue* _queue;
   Thread* _thread;
   ShenandoahTraversalGC* _traversal_gc;

@@ -1078,8 +1102,18 @@
     rp->process_discovered_references(&is_alive, &keep_alive,
                                       &complete_gc, &executor,
                                       &pt);
   }
 
-  pt.print_all_references();
-  assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
+  {
+    ShenandoahGCPhase phase(phase_process);
+    ShenandoahTerminationTracker termination(ShenandoahPhaseTimings::weakrefs_termination);
+
+    // Process leftover weak oops (using parallel version)
+    ShenandoahTraversalWeakUpdateClosure cl;
+    WeakProcessor::weak_oops_do(workers, &is_alive, &cl, 1);
+
+    pt.print_all_references();
+
+    assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
+  }
 }
< prev index next >