1 /*
   2  * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "gc/shared/referenceProcessor.hpp"
  27 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  28 #include "gc/shared/workgroup.hpp"
  29 #include "gc/shared/weakProcessor.hpp"
  30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  32 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  34 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  36 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  37 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  38 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  39 #include "gc/shenandoah/shenandoahHeuristics.hpp"
  40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  42 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  43 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  44 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  45 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  46 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  47 #include "gc/shenandoah/shenandoahUtils.hpp"
  48 #include "gc/shenandoah/shenandoahVerifier.hpp"
  49 
  50 #include "memory/iterator.hpp"
  51 #include "memory/metaspace.hpp"
  52 #include "memory/resourceArea.hpp"
  53 
  54 /**
  55  * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
  56  * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
  57  * is incremental-update-based.
  58  *
  59  * NOTE on interaction with TAMS: we want to avoid traversing new objects for
  60  * several reasons:
  61  * - We will not reclaim them in this cycle anyway, because they are not in the
  62  *   cset
  63  * - It makes up for the bulk of work during final-pause
  64  * - It also shortens the concurrent cycle because we don't need to
  65  *   pointlessly traverse through newly allocated objects.
  66  * - As a nice side-effect, it solves the I-U termination problem (mutators
  67  *   cannot outrun the GC by allocating like crazy)
  68  * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
  69  *   target object of stores if it's new. Treating new objects live implicitely
  70  *   achieves the same, but without extra barriers. I think the effect of
  71  *   shortened final-pause (mentioned above) is the main advantage of MWF. In
  72  *   particular, we will not see the head of a completely new long linked list
  73  *   in final-pause and end up traversing huge chunks of the heap there.
  74  * - We don't need to see/update the fields of new objects either, because they
  75  *   are either still null, or anything that's been stored into them has been
  76  *   evacuated+enqueued before (and will thus be treated later).
  77  *
  78  * We achieve this by setting TAMS for each region, and everything allocated
  79  * beyond TAMS will be 'implicitely marked'.
  80  *
  81  * Gotchas:
  82  * - While we want new objects to be implicitely marked, we don't want to count
  83  *   them alive. Otherwise the next cycle wouldn't pick them up and consider
  84  *   them for cset. This means that we need to protect such regions from
  85  *   getting accidentally thrashed at the end of traversal cycle. This is why I
  86  *   keep track of alloc-regions and check is_alloc_region() in the trashing
  87  *   code.
  88  * - We *need* to traverse through evacuated objects. Those objects are
  89  *   pre-existing, and any references in them point to interesting objects that
  90  *   we need to see. We also want to count them as live, because we just
  91  *   determined that they are alive :-) I achieve this by upping TAMS
  92  *   concurrently for every gclab/gc-shared alloc before publishing the
  93  *   evacuated object. This way, the GC threads will not consider such objects
  94  *   implictely marked, and traverse through them as normal.
  95  */
  96 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
  97 private:
  98   ShenandoahObjToScanQueue* _queue;
  99   ShenandoahTraversalGC* _traversal_gc;
 100   ShenandoahHeap* const _heap;
 101 
 102 public:
 103   ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
 104     _queue(q),
 105     _heap(ShenandoahHeap::heap())
 106  { }
 107 
 108   void do_buffer(void** buffer, size_t size) {
 109     for (size_t i = 0; i < size; ++i) {
 110       oop* p = (oop*) &buffer[i];
 111       oop obj = RawAccess<>::oop_load(p);
 112       shenandoah_assert_not_forwarded(p, obj);
 113       if (_heap->marking_context()->mark(obj)) {
 114         _queue->push(ShenandoahMarkTask(obj));
 115       }
 116     }
 117   }
 118 };
 119 
 120 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
 121 private:
 122   ShenandoahTraversalSATBBufferClosure* _satb_cl;
 123 
 124 public:
 125   ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
 126     _satb_cl(satb_cl) {}
 127 
 128   void do_thread(Thread* thread) {
 129     if (thread->is_Java_thread()) {
 130       JavaThread* jt = (JavaThread*)thread;
 131       ShenandoahThreadLocalData::satb_mark_queue(jt).apply_closure_and_empty(_satb_cl);
 132     } else if (thread->is_VM_thread()) {
 133       ShenandoahBarrierSet::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
 134     }
 135   }
 136 };
 137 
 138 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
 139 // and remark them later during final-traversal.
 140 class ShenandoahMarkCLDClosure : public CLDClosure {
 141 private:
 142   OopClosure* _cl;
 143 public:
 144   ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 145   void do_cld(ClassLoaderData* cld) {
 146     cld->oops_do(_cl, true, true);
 147   }
 148 };
 149 
 150 // Like CLDToOopClosure, but only process modified CLDs
 151 class ShenandoahRemarkCLDClosure : public CLDClosure {
 152 private:
 153   OopClosure* _cl;
 154 public:
 155   ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 156   void do_cld(ClassLoaderData* cld) {
 157     if (cld->has_modified_oops()) {
 158       cld->oops_do(_cl, true, true);
 159     }
 160   }
 161 };
 162 
 163 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
 164 private:
 165   ShenandoahCSetRootScanner* _rp;
 166   ShenandoahHeap* _heap;
 167   ShenandoahCsetCodeRootsIterator* _cset_coderoots;
 168   ShenandoahStringDedupRoots       _dedup_roots;
 169 
 170 public:
 171   ShenandoahInitTraversalCollectionTask(ShenandoahCSetRootScanner* rp) :
 172     AbstractGangTask("Shenandoah Init Traversal Collection"),
 173     _rp(rp),
 174     _heap(ShenandoahHeap::heap()) {}
 175 
 176   void work(uint worker_id) {
 177     ShenandoahParallelWorkerSession worker_session(worker_id);
 178 
 179     ShenandoahEvacOOMScope oom_evac_scope;
 180     ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
 181     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 182 
 183     bool process_refs = _heap->process_references();
 184     bool unload_classes = _heap->unload_classes();
 185     ReferenceProcessor* rp = NULL;
 186     if (process_refs) {
 187       rp = _heap->ref_processor();
 188     }
 189 
 190     // Step 1: Process ordinary GC roots.
 191     {
 192       ShenandoahTraversalClosure roots_cl(q, rp);
 193       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
 194       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 195       if (unload_classes) {
 196         _rp->roots_do(worker_id, &roots_cl, NULL, &code_cl);
 197       } else {
 198         _rp->roots_do(worker_id, &roots_cl, &cld_cl, &code_cl);
 199       }
 200 
 201       AlwaysTrueClosure is_alive;
 202       _dedup_roots.oops_do(&is_alive, &roots_cl, worker_id);
 203     }
 204   }
 205 };
 206 
 207 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 208 private:
 209   ShenandoahTaskTerminator* _terminator;
 210   ShenandoahHeap* _heap;
 211 public:
 212   ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) :
 213     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 214     _terminator(terminator),
 215     _heap(ShenandoahHeap::heap()) {}
 216 
 217   void work(uint worker_id) {
 218     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 219     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 220     ShenandoahEvacOOMScope oom_evac_scope;
 221     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 222 
 223     // Drain all outstanding work in queues.
 224     traversal_gc->main_loop(worker_id, _terminator, true);
 225   }
 226 };
 227 
 228 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 229 private:
 230   ShenandoahAllRootScanner* _rp;
 231   ShenandoahTaskTerminator* _terminator;
 232   ShenandoahHeap* _heap;
 233 public:
 234   ShenandoahFinalTraversalCollectionTask(ShenandoahAllRootScanner* rp, ShenandoahTaskTerminator* terminator) :
 235     AbstractGangTask("Shenandoah Final Traversal Collection"),
 236     _rp(rp),
 237     _terminator(terminator),
 238     _heap(ShenandoahHeap::heap()) {}
 239 
 240   void work(uint worker_id) {
 241     ShenandoahParallelWorkerSession worker_session(worker_id);
 242 
 243     ShenandoahEvacOOMScope oom_evac_scope;
 244     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 245 
 246     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 247     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 248 
 249     bool process_refs = _heap->process_references();
 250     bool unload_classes = _heap->unload_classes();
 251     ReferenceProcessor* rp = NULL;
 252     if (process_refs) {
 253       rp = _heap->ref_processor();
 254     }
 255 
 256     // Step 0: Drain outstanding SATB queues.
 257     // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
 258     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 259     {
 260       // Process remaining finished SATB buffers.
 261       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 262       while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
 263       // Process remaining threads SATB buffers below.
 264     }
 265 
 266     // Step 1: Process GC roots.
 267     // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
 268     // and the references to the oops are updated during init pause. New nmethods are handled
 269     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
 270     // roots here.
 271     if (!_heap->is_degenerated_gc_in_progress()) {
 272       ShenandoahTraversalClosure roots_cl(q, rp);
 273       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 274       if (unload_classes) {
 275         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 276         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 277       } else {
 278         CLDToOopClosure cld_cl(&roots_cl);
 279         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 280       }
 281     } else {
 282       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 283       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 284       if (unload_classes) {
 285         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 286         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 287       } else {
 288         CLDToOopClosure cld_cl(&roots_cl);
 289         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 290       }
 291     }
 292 
 293     {
 294       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 295       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 296 
 297       // Step 3: Finally drain all outstanding work in queues.
 298       traversal_gc->main_loop(worker_id, _terminator, false);
 299     }
 300 
 301   }
 302 };
 303 
 304 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 305   _heap(heap),
 306   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
 307   _traversal_set(ShenandoahHeapRegionSet()) {
 308 
 309   // Traversal does not support concurrent code root scanning
 310   FLAG_SET_DEFAULT(ShenandoahConcurrentScanCodeRoots, false);
 311 
 312   uint num_queues = heap->max_workers();
 313   for (uint i = 0; i < num_queues; ++i) {
 314     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 315     task_queue->initialize();
 316     _task_queues->register_queue(i, task_queue);
 317   }
 318 }
 319 
 320 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
 321 }
 322 
 323 void ShenandoahTraversalGC::prepare_regions() {
 324   size_t num_regions = _heap->num_regions();
 325   ShenandoahMarkingContext* const ctx = _heap->marking_context();
 326   for (size_t i = 0; i < num_regions; i++) {
 327     ShenandoahHeapRegion* region = _heap->get_region(i);
 328     if (_heap->is_bitmap_slice_committed(region)) {
 329       if (_traversal_set.is_in(i)) {
 330         ctx->capture_top_at_mark_start(region);
 331         region->clear_live_data();
 332         assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared");
 333       } else {
 334         // Everything outside the traversal set is always considered live.
 335         ctx->reset_top_at_mark_start(region);
 336       }
 337     } else {
 338       // FreeSet may contain uncommitted empty regions, once they are recommitted,
 339       // their TAMS may have old values, so reset them here.
 340       ctx->reset_top_at_mark_start(region);
 341     }
 342   }
 343 }
 344 
 345 void ShenandoahTraversalGC::prepare() {
 346   _heap->collection_set()->clear();
 347   assert(_heap->collection_set()->count() == 0, "collection set not clear");
 348 
 349   if (UseTLAB) {
 350     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_accumulate_stats);
 351     _heap->accumulate_statistics_tlabs();
 352   }
 353 
 354   {
 355     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
 356     _heap->make_parsable(true);
 357   }
 358 
 359   if (UseTLAB) {
 360     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs);
 361     _heap->resize_tlabs();
 362   }
 363 
 364   assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
 365   assert(!_heap->marking_context()->is_complete(), "should not be complete");
 366 
 367   ShenandoahFreeSet* free_set = _heap->free_set();
 368   ShenandoahCollectionSet* collection_set = _heap->collection_set();
 369 
 370   // Find collection set
 371   _heap->heuristics()->choose_collection_set(collection_set);
 372   prepare_regions();
 373 
 374   // Rebuild free set
 375   free_set->rebuild();
 376 
 377   log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "M, " SIZE_FORMAT "M CSet, " SIZE_FORMAT " CSet regions",
 378                      collection_set->garbage() / M, collection_set->live_data() / M, collection_set->count());
 379 }
 380 
 381 void ShenandoahTraversalGC::init_traversal_collection() {
 382   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
 383 
 384   if (ShenandoahVerify) {
 385     _heap->verifier()->verify_before_traversal();
 386   }
 387 
 388   if (VerifyBeforeGC) {
 389     Universe::verify();
 390   }
 391 
 392   {
 393     ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
 394     ShenandoahHeapLocker lock(_heap->lock());
 395     prepare();
 396   }
 397 
 398   _heap->set_concurrent_traversal_in_progress(true);
 399 
 400   bool process_refs = _heap->process_references();
 401   if (process_refs) {
 402     ReferenceProcessor* rp = _heap->ref_processor();
 403     rp->enable_discovery(true /*verify_no_refs*/);
 404     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 405   }
 406 
 407   {
 408     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
 409     assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
 410     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 411 
 412 #if COMPILER2_OR_JVMCI
 413     DerivedPointerTable::clear();
 414 #endif
 415 
 416     {
 417       uint nworkers = _heap->workers()->active_workers();
 418       task_queues()->reserve(nworkers);
 419       ShenandoahCSetRootScanner rp(nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
 420       ShenandoahInitTraversalCollectionTask traversal_task(&rp);
 421       _heap->workers()->run_task(&traversal_task);
 422     }
 423 
 424 #if COMPILER2_OR_JVMCI
 425     DerivedPointerTable::update_pointers();
 426 #endif
 427   }
 428 
 429   if (ShenandoahPacing) {
 430     _heap->pacer()->setup_for_traversal();
 431   }
 432 }
 433 
 434 void ShenandoahTraversalGC::main_loop(uint w, ShenandoahTaskTerminator* t, bool sts_yield) {
 435   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 436 
 437   // Initialize live data.
 438   jushort* ld = _heap->get_liveness_cache(w);
 439 
 440   ReferenceProcessor* rp = NULL;
 441   if (_heap->process_references()) {
 442     rp = _heap->ref_processor();
 443   }
 444   {
 445     if (!_heap->is_degenerated_gc_in_progress()) {
 446       if (_heap->unload_classes()) {
 447         if (ShenandoahStringDedup::is_enabled()) {
 448           ShenandoahTraversalMetadataDedupClosure cl(q, rp);
 449           main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t, sts_yield);
 450         } else {
 451           ShenandoahTraversalMetadataClosure cl(q, rp);
 452           main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t, sts_yield);
 453         }
 454       } else {
 455         if (ShenandoahStringDedup::is_enabled()) {
 456           ShenandoahTraversalDedupClosure cl(q, rp);
 457           main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t, sts_yield);
 458         } else {
 459           ShenandoahTraversalClosure cl(q, rp);
 460           main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t, sts_yield);
 461         }
 462       }
 463     } else {
 464       if (_heap->unload_classes()) {
 465         if (ShenandoahStringDedup::is_enabled()) {
 466           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp);
 467           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 468         } else {
 469           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 470           main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t, sts_yield);
 471         }
 472       } else {
 473         if (ShenandoahStringDedup::is_enabled()) {
 474           ShenandoahTraversalDedupDegenClosure cl(q, rp);
 475           main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 476         } else {
 477           ShenandoahTraversalDegenClosure cl(q, rp);
 478           main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t, sts_yield);
 479         }
 480       }
 481     }
 482   }
 483 
 484   _heap->flush_liveness_cache(w);
 485 }
 486 
 487 template <class T>
 488 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator* terminator, bool sts_yield) {
 489   ShenandoahObjToScanQueueSet* queues = task_queues();
 490   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 491   ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark();
 492 
 493   uintx stride = ShenandoahMarkLoopStride;
 494 
 495   ShenandoahMarkTask task;
 496 
 497   // Process outstanding queues, if any.
 498   q = queues->claim_next();
 499   while (q != NULL) {
 500     if (_heap->check_cancelled_gc_and_yield(sts_yield)) {
 501       return;
 502     }
 503 
 504     for (uint i = 0; i < stride; i++) {
 505       if (q->pop(task)) {
 506         conc_mark->do_task<T>(q, cl, live_data, &task);
 507       } else {
 508         assert(q->is_empty(), "Must be empty");
 509         q = queues->claim_next();
 510         break;
 511       }
 512     }
 513   }
 514 
 515   if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 516 
 517   // Normal loop.
 518   q = queues->queue(worker_id);
 519 
 520   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 521   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 522 
 523   int seed = 17;
 524 
 525   while (true) {
 526     if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 527 
 528     while (satb_mq_set.completed_buffers_num() > 0) {
 529       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 530     }
 531 
 532     uint work = 0;
 533     for (uint i = 0; i < stride; i++) {
 534       if (q->pop(task) ||
 535           queues->steal(worker_id, &seed, task)) {
 536         conc_mark->do_task<T>(q, cl, live_data, &task);
 537         work++;
 538       } else {
 539         break;
 540       }
 541     }
 542 
 543     if (work == 0) {
 544       // No more work, try to terminate
 545       ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 546       ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
 547       ShenandoahTerminationTimingsTracker term_tracker(worker_id);
 548       ShenandoahTerminatorTerminator tt(_heap);
 549 
 550       if (terminator->offer_termination(&tt)) return;
 551     }
 552   }
 553 }
 554 
 555 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ShenandoahTaskTerminator* terminator, bool sts_yield) {
 556   if (_heap->cancelled_gc()) {
 557     return true;
 558   }
 559   return false;
 560 }
 561 
 562 void ShenandoahTraversalGC::concurrent_traversal_collection() {
 563   ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
 564   if (!_heap->cancelled_gc()) {
 565     uint nworkers = _heap->workers()->active_workers();
 566     task_queues()->reserve(nworkers);
 567     ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination);
 568 
 569     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 570     ShenandoahConcurrentTraversalCollectionTask task(&terminator);
 571     _heap->workers()->run_task(&task);
 572   }
 573 
 574   if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) {
 575     preclean_weak_refs();
 576   }
 577 }
 578 
 579 void ShenandoahTraversalGC::final_traversal_collection() {
 580   _heap->make_parsable(true);
 581 
 582   if (!_heap->cancelled_gc()) {
 583 #if COMPILER2_OR_JVMCI
 584     DerivedPointerTable::clear();
 585 #endif
 586     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
 587     uint nworkers = _heap->workers()->active_workers();
 588     task_queues()->reserve(nworkers);
 589 
 590     // Finish traversal
 591     ShenandoahAllRootScanner rp(nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
 592     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
 593 
 594     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 595     ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
 596     _heap->workers()->run_task(&task);
 597 #if COMPILER2_OR_JVMCI
 598     DerivedPointerTable::update_pointers();
 599 #endif
 600   }
 601 
 602   if (!_heap->cancelled_gc() && _heap->process_references()) {
 603     weak_refs_work();
 604   }
 605 
 606   if (!_heap->cancelled_gc()) {
 607     fixup_roots();
 608     if (_heap->unload_classes()) {
 609       _heap->unload_classes_and_cleanup_tables(false);
 610     } else {
 611       ShenandoahIsAliveSelector alive;
 612       StringTable::unlink(alive.is_alive_closure());
 613     }
 614   }
 615 
 616   if (!_heap->cancelled_gc()) {
 617     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 618     TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
 619     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 620 
 621     // No more marking expected
 622     _heap->mark_complete_marking_context();
 623 
 624     // Resize metaspace
 625     MetaspaceGC::compute_new_size();
 626 
 627     // Still good? We can now trash the cset, and make final verification
 628     {
 629       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
 630       ShenandoahHeapLocker lock(_heap->lock());
 631 
 632       // Trash everything
 633       // Clear immediate garbage regions.
 634       size_t num_regions = _heap->num_regions();
 635 
 636       ShenandoahHeapRegionSet* traversal_regions = traversal_set();
 637       ShenandoahFreeSet* free_regions = _heap->free_set();
 638       ShenandoahMarkingContext* const ctx = _heap->marking_context();
 639       free_regions->clear();
 640       for (size_t i = 0; i < num_regions; i++) {
 641         ShenandoahHeapRegion* r = _heap->get_region(i);
 642         bool not_allocated = ctx->top_at_mark_start(r) == r->top();
 643 
 644         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
 645         if (r->is_humongous_start() && candidate) {
 646           // Trash humongous.
 647           HeapWord* humongous_obj = r->bottom();
 648           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
 649           r->make_trash_immediate();
 650           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
 651             i++;
 652             r = _heap->get_region(i);
 653             assert(r->is_humongous_continuation(), "must be humongous continuation");
 654             r->make_trash_immediate();
 655           }
 656         } else if (!r->is_empty() && candidate) {
 657           // Trash regular.
 658           assert(!r->is_humongous(), "handled above");
 659           assert(!r->is_trash(), "must not already be trashed");
 660           r->make_trash_immediate();
 661         }
 662       }
 663       _heap->collection_set()->clear();
 664       _heap->free_set()->rebuild();
 665       reset();
 666     }
 667 
 668     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 669     _heap->set_concurrent_traversal_in_progress(false);
 670     assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here");
 671 
 672     if (ShenandoahVerify) {
 673       _heap->verifier()->verify_after_traversal();
 674     }
 675 
 676     if (VerifyAfterGC) {
 677       Universe::verify();
 678     }
 679   }
 680 }
 681 
 682 class ShenandoahTraversalFixRootsClosure : public OopClosure {
 683 private:
 684   template <class T>
 685   inline void do_oop_work(T* p) {
 686     T o = RawAccess<>::oop_load(p);
 687     if (!CompressedOops::is_null(o)) {
 688       oop obj = CompressedOops::decode_not_null(o);
 689       oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 690       if (obj != forw) {
 691         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 692       }
 693     }
 694   }
 695 
 696 public:
 697   inline void do_oop(oop* p) { do_oop_work(p); }
 698   inline void do_oop(narrowOop* p) { do_oop_work(p); }
 699 };
 700 
 701 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
 702 private:
 703   ShenandoahRootUpdater* _rp;
 704 
 705 public:
 706   ShenandoahTraversalFixRootsTask(ShenandoahRootUpdater* rp) :
 707     AbstractGangTask("Shenandoah traversal fix roots"),
 708     _rp(rp) {
 709     assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
 710   }
 711 
 712   void work(uint worker_id) {
 713     ShenandoahParallelWorkerSession worker_session(worker_id);
 714     ShenandoahTraversalFixRootsClosure cl;
 715     ShenandoahForwardedIsAliveClosure is_alive;
 716     _rp->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahTraversalFixRootsClosure>(worker_id, &is_alive, &cl);
 717   }
 718 };
 719 
 720 void ShenandoahTraversalGC::fixup_roots() {
 721 #if COMPILER2_OR_JVMCI
 722   DerivedPointerTable::clear();
 723 #endif
 724   ShenandoahRootUpdater rp(_heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots, true /* update code cache */);
 725   ShenandoahTraversalFixRootsTask update_roots_task(&rp);
 726   _heap->workers()->run_task(&update_roots_task);
 727 #if COMPILER2_OR_JVMCI
 728   DerivedPointerTable::update_pointers();
 729 #endif
 730 }
 731 
 732 void ShenandoahTraversalGC::reset() {
 733   _task_queues->clear();
 734 }
 735 
 736 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
 737   return _task_queues;
 738 }
 739 
 740 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 741 private:
 742   ShenandoahHeap* const _heap;
 743 public:
 744   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 745   virtual bool should_return() { return _heap->cancelled_gc(); }
 746 };
 747 
 748 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 749 public:
 750   void do_void() {
 751     ShenandoahHeap* sh = ShenandoahHeap::heap();
 752     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 753     assert(sh->process_references(), "why else would we be here?");
 754     ShenandoahTaskTerminator terminator(1, traversal_gc->task_queues());
 755     shenandoah_assert_rp_isalive_installed();
 756     traversal_gc->main_loop((uint) 0, &terminator, true);
 757   }
 758 };
 759 
 760 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 761 private:
 762   ShenandoahObjToScanQueue* _queue;
 763   Thread* _thread;
 764   ShenandoahTraversalGC* _traversal_gc;
 765   ShenandoahMarkingContext* const _mark_context;
 766 
 767   template <class T>
 768   inline void do_oop_work(T* p) {
 769     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
 770   }
 771 
 772 public:
 773   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 774     _queue(q), _thread(Thread::current()),
 775     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 776     _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 777 
 778   void do_oop(narrowOop* p) { do_oop_work(p); }
 779   void do_oop(oop* p)       { do_oop_work(p); }
 780 };
 781 
 782 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
 783 private:
 784   ShenandoahObjToScanQueue* _queue;
 785   Thread* _thread;
 786   ShenandoahTraversalGC* _traversal_gc;
 787   ShenandoahMarkingContext* const _mark_context;
 788 
 789   template <class T>
 790   inline void do_oop_work(T* p) {
 791     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
 792   }
 793 
 794 public:
 795   ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 796           _queue(q), _thread(Thread::current()),
 797           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 798           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 799 
 800   void do_oop(narrowOop* p) { do_oop_work(p); }
 801   void do_oop(oop* p)       { do_oop_work(p); }
 802 };
 803 
 804 class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure {
 805 private:
 806   ShenandoahObjToScanQueue* _queue;
 807   Thread* _thread;
 808   ShenandoahTraversalGC* _traversal_gc;
 809   ShenandoahMarkingContext* const _mark_context;
 810 
 811   template <class T>
 812   inline void do_oop_work(T* p) {
 813     ShenandoahEvacOOMScope evac_scope;
 814     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
 815   }
 816 
 817 public:
 818   ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 819           _queue(q), _thread(Thread::current()),
 820           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 821           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 822 
 823   void do_oop(narrowOop* p) { do_oop_work(p); }
 824   void do_oop(oop* p)       { do_oop_work(p); }
 825 };
 826 
 827 class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure {
 828 private:
 829   ShenandoahObjToScanQueue* _queue;
 830   Thread* _thread;
 831   ShenandoahTraversalGC* _traversal_gc;
 832   ShenandoahMarkingContext* const _mark_context;
 833 
 834   template <class T>
 835   inline void do_oop_work(T* p) {
 836     ShenandoahEvacOOMScope evac_scope;
 837     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
 838   }
 839 
 840 public:
 841   ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 842           _queue(q), _thread(Thread::current()),
 843           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 844           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 845 
 846   void do_oop(narrowOop* p) { do_oop_work(p); }
 847   void do_oop(oop* p)       { do_oop_work(p); }
 848 };
 849 
 850 class ShenandoahTraversalPrecleanTask : public AbstractGangTask {
 851 private:
 852   ReferenceProcessor* _rp;
 853 
 854 public:
 855   ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) :
 856           AbstractGangTask("Precleaning task"),
 857           _rp(rp) {}
 858 
 859   void work(uint worker_id) {
 860     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 861     ShenandoahParallelWorkerSession worker_session(worker_id);
 862     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 863     ShenandoahEvacOOMScope oom_evac_scope;
 864 
 865     ShenandoahHeap* sh = ShenandoahHeap::heap();
 866 
 867     ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id);
 868 
 869     ShenandoahForwardedIsAliveClosure is_alive;
 870     ShenandoahTraversalCancelledGCYieldClosure yield;
 871     ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
 872     ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q);
 873     ResourceMark rm;
 874     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 875                                         &complete_gc, &yield,
 876                                         NULL);
 877   }
 878 };
 879 
 880 void ShenandoahTraversalGC::preclean_weak_refs() {
 881   // Pre-cleaning weak references before diving into STW makes sense at the
 882   // end of concurrent mark. This will filter out the references which referents
 883   // are alive. Note that ReferenceProcessor already filters out these on reference
 884   // discovery, and the bulk of work is done here. This phase processes leftovers
 885   // that missed the initial filtering, i.e. when referent was marked alive after
 886   // reference was discovered by RP.
 887 
 888   assert(_heap->process_references(), "sanity");
 889   assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
 890 
 891   // Shortcut if no references were discovered to avoid winding up threads.
 892   ReferenceProcessor* rp = _heap->ref_processor();
 893   if (!rp->has_discovered_references()) {
 894     return;
 895   }
 896 
 897   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 898 
 899   shenandoah_assert_rp_isalive_not_installed();
 900   ShenandoahForwardedIsAliveClosure is_alive;
 901   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
 902 
 903   assert(task_queues()->is_empty(), "Should be empty");
 904 
 905   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 906   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 907   // parallel precleans, we can extend this to more threads.
 908   ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false);
 909 
 910   WorkGang* workers = _heap->workers();
 911   uint nworkers = workers->active_workers();
 912   assert(nworkers == 1, "This code uses only a single worker");
 913   task_queues()->reserve(nworkers);
 914 
 915   ShenandoahTraversalPrecleanTask task(rp);
 916   workers->run_task(&task);
 917 
 918   assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty");
 919 }
 920 
 921 // Weak Reference Closures
 922 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 923   uint _worker_id;
 924   ShenandoahTaskTerminator* _terminator;
 925   bool _reset_terminator;
 926 
 927 public:
 928   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 929     _worker_id(worker_id),
 930     _terminator(t),
 931     _reset_terminator(reset_terminator) {
 932   }
 933 
 934   void do_void() {
 935     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 936 
 937     ShenandoahHeap* sh = ShenandoahHeap::heap();
 938     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 939     assert(sh->process_references(), "why else would we be here?");
 940     shenandoah_assert_rp_isalive_installed();
 941 
 942     traversal_gc->main_loop(_worker_id, _terminator, false);
 943 
 944     if (_reset_terminator) {
 945       _terminator->reset_for_reuse();
 946     }
 947   }
 948 };
 949 
 950 class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure {
 951   uint _worker_id;
 952   ShenandoahTaskTerminator* _terminator;
 953   bool _reset_terminator;
 954 
 955 public:
 956   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 957           _worker_id(worker_id),
 958           _terminator(t),
 959           _reset_terminator(reset_terminator) {
 960   }
 961 
 962   void do_void() {
 963     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 964 
 965     ShenandoahHeap* sh = ShenandoahHeap::heap();
 966     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 967     assert(sh->process_references(), "why else would we be here?");
 968     shenandoah_assert_rp_isalive_installed();
 969 
 970     ShenandoahEvacOOMScope evac_scope;
 971     traversal_gc->main_loop(_worker_id, _terminator, false);
 972 
 973     if (_reset_terminator) {
 974       _terminator->reset_for_reuse();
 975     }
 976   }
 977 };
 978 
 979 void ShenandoahTraversalGC::weak_refs_work() {
 980   assert(_heap->process_references(), "sanity");
 981 
 982   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
 983 
 984   ShenandoahGCPhase phase(phase_root);
 985 
 986   ReferenceProcessor* rp = _heap->ref_processor();
 987 
 988   // NOTE: We cannot shortcut on has_discovered_references() here, because
 989   // we will miss marking JNI Weak refs then, see implementation in
 990   // ReferenceProcessor::process_discovered_references.
 991   weak_refs_work_doit();
 992 
 993   rp->verify_no_references_recorded();
 994   assert(!rp->discovery_enabled(), "Post condition");
 995 
 996 }
 997 
 998 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
 999 private:
1000   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
1001   ShenandoahTaskTerminator* _terminator;
1002 
1003 public:
1004   ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
1005                                       ShenandoahTaskTerminator* t) :
1006     AbstractGangTask("Process reference objects in parallel"),
1007     _proc_task(proc_task),
1008     _terminator(t) {
1009   }
1010 
1011   void work(uint worker_id) {
1012     ShenandoahEvacOOMScope oom_evac_scope;
1013     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1014     ShenandoahHeap* heap = ShenandoahHeap::heap();
1015     ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
1016 
1017     ShenandoahForwardedIsAliveClosure is_alive;
1018     if (!heap->is_degenerated_gc_in_progress()) {
1019       ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1020       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1021     } else {
1022       ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1023       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1024     }
1025   }
1026 };
1027 
1028 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
1029 private:
1030   WorkGang* _workers;
1031 
1032 public:
1033   ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {}
1034 
1035   // Executes a task using worker threads.
1036   void execute(ProcessTask& task, uint ergo_workers) {
1037     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1038 
1039     ShenandoahHeap* heap = ShenandoahHeap::heap();
1040     ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
1041     ShenandoahPushWorkerQueuesScope scope(_workers,
1042                                           traversal_gc->task_queues(),
1043                                           ergo_workers,
1044                                           /* do_check = */ false);
1045     uint nworkers = _workers->active_workers();
1046     traversal_gc->task_queues()->reserve(nworkers);
1047     ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues());
1048     ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
1049     _workers->run_task(&proc_task_proxy);
1050   }
1051 };
1052 
1053 void ShenandoahTraversalGC::weak_refs_work_doit() {
1054   ReferenceProcessor* rp = _heap->ref_processor();
1055 
1056   ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
1057 
1058   shenandoah_assert_rp_isalive_not_installed();
1059   ShenandoahForwardedIsAliveClosure is_alive;
1060   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
1061 
1062   WorkGang* workers = _heap->workers();
1063   uint nworkers = workers->active_workers();
1064 
1065   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
1066   rp->set_active_mt_degree(nworkers);
1067 
1068   assert(task_queues()->is_empty(), "Should be empty");
1069 
1070   // complete_gc and keep_alive closures instantiated here are only needed for
1071   // single-threaded path in RP. They share the queue 0 for tracking work, which
1072   // simplifies implementation. Since RP may decide to call complete_gc several
1073   // times, we need to be able to reuse the terminator.
1074   uint serial_worker_id = 0;
1075   ShenandoahTaskTerminator terminator(1, task_queues());
1076   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
1077   ShenandoahPushWorkerQueuesScope scope(workers, task_queues(), 1, /* do_check = */ false);
1078 
1079   ShenandoahTraversalRefProcTaskExecutor executor(workers);
1080 
1081   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
1082   if (!_heap->is_degenerated_gc_in_progress()) {
1083     ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
1084     rp->process_discovered_references(&is_alive, &keep_alive,
1085                                       &complete_gc, &executor,
1086                                       &pt);
1087   } else {
1088     ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id));
1089     rp->process_discovered_references(&is_alive, &keep_alive,
1090                                       &complete_gc, &executor,
1091                                       &pt);
1092   }
1093 
1094   pt.print_all_references();
1095   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
1096 }