1 /*
   2  * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "classfile/symbolTable.hpp"
  27 #include "classfile/systemDictionary.hpp"
  28 #include "code/codeCache.hpp"
  29 
  30 #include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp"
  31 #include "gc_implementation/shenandoah/shenandoahClosures.inline.hpp"
  32 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
  33 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
  34 #include "gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp"
  35 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc_implementation/shenandoah/shenandoahParallelCleaning.hpp"
  37 #include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
  38 #include "gc_implementation/shenandoah/shenandoahRootProcessor.inline.hpp"
  39 #include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp"
  40 #include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
  41 #include "gc_implementation/shenandoah/shenandoahUtils.hpp"
  42 #include "gc_implementation/shenandoah/shenandoah_specialized_oop_closures.hpp"
  43 
  44 #include "memory/referenceProcessor.hpp"
  45 #include "memory/iterator.inline.hpp"
  46 #include "memory/metaspace.hpp"
  47 #include "memory/resourceArea.hpp"
  48 #include "oops/oop.inline.hpp"
  49 
  50 template<UpdateRefsMode UPDATE_REFS>
  51 class ShenandoahInitMarkRootsClosure : public OopClosure {
  52 private:
  53   ShenandoahObjToScanQueue* _queue;
  54   ShenandoahHeap* _heap;
  55   ShenandoahStrDedupQueue*  _dedup_queue;
  56   ShenandoahMarkingContext* const _mark_context;
  57 
  58   template <class T>
  59   inline void do_oop_nv(T* p) {
  60     ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context, _dedup_queue);
  61   }
  62 
  63 public:
  64   ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq) :
  65     _queue(q),
  66     _heap(ShenandoahHeap::heap()),
  67     _dedup_queue(dq),
  68     _mark_context(_heap->marking_context()) {};
  69 
  70   void do_oop(narrowOop* p) { do_oop_nv(p); }
  71   void do_oop(oop* p)       { do_oop_nv(p); }
  72 };
  73 
  74 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
  75   MetadataAwareOopClosure(rp),
  76   _queue(q),
  77   _dedup_queue(NULL),
  78   _heap(ShenandoahHeap::heap()),
  79   _mark_context(_heap->marking_context())
  80 { }
  81 
  82 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) :
  83   MetadataAwareOopClosure(rp),
  84   _queue(q),
  85   _dedup_queue(dq),
  86   _heap(ShenandoahHeap::heap()),
  87   _mark_context(_heap->marking_context())
  88 { }
  89 
  90 template<UpdateRefsMode UPDATE_REFS>
  91 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
  92 private:
  93   ShenandoahAllRootScanner* _rp;
  94 public:
  95   ShenandoahInitMarkRootsTask(ShenandoahAllRootScanner* rp) :
  96     AbstractGangTask("Shenandoah init mark roots task"),
  97     _rp(rp) {
  98   }
  99 
 100   void work(uint worker_id) {
 101     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 102     ShenandoahParallelWorkerSession worker_session(worker_id);
 103 
 104     ShenandoahHeap* heap = ShenandoahHeap::heap();
 105     ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
 106     assert(queues->get_reserved() > worker_id, err_msg("Queue has not been reserved for worker id: %d", worker_id));
 107 
 108     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 109     ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q, NULL);
 110     do_work(heap, &mark_cl, worker_id);
 111   }
 112 
 113 private:
 114   void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
 115     // The rationale for selecting the roots to scan is as follows:
 116     //   a. With unload_classes = true, we only want to scan the actual strong roots from the
 117     //      code cache. This will allow us to identify the dead classes, unload them, *and*
 118     //      invalidate the relevant code cache blobs. This could be only done together with
 119     //      class unloading.
 120     //   b. With unload_classes = false, we have to nominally retain all the references from code
 121     //      cache, because there could be the case of embedded class/oop in the generated code,
 122     //      which we will never visit during mark. Without code cache invalidation, as in (a),
 123     //      we risk executing that code cache blob, and crashing.
 124     //   c. With ShenandoahConcurrentScanCodeRoots, we avoid scanning the entire code cache here,
 125     //      and instead do that in concurrent phase under the relevant lock. This saves init mark
 126     //      pause time.
 127     ResourceMark m;
 128     if (heap->unload_classes()) {
 129       _rp->strong_roots_do(worker_id, oops);
 130     } else {
 131       _rp->roots_do(worker_id, oops);
 132     }
 133   }
 134 };
 135 
 136 class ShenandoahUpdateRootsTask : public AbstractGangTask {
 137 private:
 138   ShenandoahRootUpdater* _root_updater;
 139 public:
 140   ShenandoahUpdateRootsTask(ShenandoahRootUpdater* _root_updater) :
 141     AbstractGangTask("Shenandoah update roots task"),
 142     _root_updater(_root_updater) {
 143   }
 144 
 145   void work(uint worker_id) {
 146     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 147     ShenandoahParallelWorkerSession worker_session(worker_id);
 148 
 149     ShenandoahHeap* heap = ShenandoahHeap::heap();
 150     ShenandoahUpdateRefsClosure cl;
 151     ShenandoahIsAliveSelector is_alive;
 152     _root_updater->roots_do(worker_id, is_alive.is_alive_closure(), &cl);
 153   }
 154 };
 155 
 156 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
 157 private:
 158   ShenandoahConcurrentMark* _cm;
 159   ShenandoahTaskTerminator* _terminator;
 160 
 161 public:
 162   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator) :
 163     AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
 164   }
 165 
 166   void work(uint worker_id) {
 167     ShenandoahHeap* heap = ShenandoahHeap::heap();
 168     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 169     ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 170     ReferenceProcessor* rp;
 171     if (heap->process_references()) {
 172       rp = heap->ref_processor();
 173       shenandoah_assert_rp_isalive_installed();
 174     } else {
 175       rp = NULL;
 176     }
 177 
 178     _cm->concurrent_scan_code_roots(worker_id, rp);
 179     _cm->mark_loop(worker_id, _terminator, rp,
 180                    true, // cancellable
 181                    ShenandoahStringDedup::is_enabled()); // perform string dedup
 182   }
 183 };
 184 
 185 class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure {
 186 private:
 187   ShenandoahSATBBufferClosure* _satb_cl;
 188   OopClosure*            const _cl;
 189   MarkingCodeBlobClosure*      _code_cl;
 190   int _thread_parity;
 191 
 192 public:
 193   ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) :
 194     _satb_cl(satb_cl), _cl(cl), _code_cl(code_cl),
 195     _thread_parity(SharedHeap::heap()->strong_roots_parity()) {}
 196 
 197   void do_thread(Thread* thread) {
 198     if (thread->is_Java_thread()) {
 199       if (thread->claim_oops_do(true, _thread_parity)) {
 200         JavaThread* jt = (JavaThread*)thread;
 201         jt->satb_mark_queue().apply_closure_and_empty(_satb_cl);
 202         if (_cl != NULL) {
 203           ResourceMark rm;
 204           jt->oops_do(_cl, NULL, _code_cl);
 205         } else if (_code_cl != NULL) {
 206           // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
 207           // however the liveness of oops reachable from nmethods have very complex lifecycles:
 208           // * Alive if on the stack of an executing method
 209           // * Weakly reachable otherwise
 210           // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
 211           // live by the SATB invariant but other oops recorded in nmethods may behave differently.
 212           jt->nmethods_do(_code_cl);
 213         }
 214       }
 215     } else if (thread->is_VM_thread()) {
 216       if (thread->claim_oops_do(true, _thread_parity)) {
 217         JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
 218       }
 219     }
 220   }
 221 };
 222 
 223 class ShenandoahFinalMarkingTask : public AbstractGangTask {
 224 private:
 225   ShenandoahConcurrentMark* _cm;
 226   ShenandoahTaskTerminator* _terminator;
 227   bool                      _dedup_string;
 228   ShenandoahSharedFlag      _claimed_syncroots;
 229 
 230 public:
 231   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator, bool dedup_string) :
 232     AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
 233   }
 234 
 235   void work(uint worker_id) {
 236     ShenandoahHeap* heap = ShenandoahHeap::heap();
 237 
 238     ReferenceProcessor* rp;
 239     if (heap->process_references()) {
 240       rp = heap->ref_processor();
 241       shenandoah_assert_rp_isalive_installed();
 242     } else {
 243       rp = NULL;
 244     }
 245 
 246     // First drain remaining SATB buffers.
 247     // Notice that this is not strictly necessary for mark-compact. But since
 248     // it requires a StrongRootsScope around the task, we need to claim the
 249     // threads, and performance-wise it doesn't really matter. Adds about 1ms to
 250     // full-gc.
 251     {
 252       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 253       ShenandoahStrDedupQueue *dq = NULL;
 254       if (ShenandoahStringDedup::is_enabled()) {
 255          dq = ShenandoahStringDedup::queue(worker_id);
 256       }
 257       ShenandoahSATBBufferClosure cl(q, dq);
 258       SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
 259       while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
 260       bool do_nmethods = heap->unload_classes();
 261       if (heap->has_forwarded_objects()) {
 262         ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
 263         MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
 264         ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
 265                                                           ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL,
 266                                                           do_nmethods ? &blobsCl : NULL);
 267         Threads::threads_do(&tc);
 268         if (ShenandoahStoreValEnqueueBarrier && _claimed_syncroots.try_set()) {
 269           ObjectSynchronizer::oops_do(&resolve_mark_cl);
 270         }
 271       } else {
 272         ShenandoahMarkRefsClosure mark_cl(q, rp);
 273         MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
 274         ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
 275                                                           ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL,
 276                                                           do_nmethods ? &blobsCl : NULL);
 277         Threads::threads_do(&tc);
 278         if (ShenandoahStoreValEnqueueBarrier && _claimed_syncroots.try_set()) {
 279           ObjectSynchronizer::oops_do(&mark_cl);
 280         }
 281       }
 282     }
 283 
 284     if (heap->is_degenerated_gc_in_progress() || heap->is_full_gc_in_progress()) {
 285       // Full GC does not execute concurrent cycle.
 286       // Degenerated cycle may bypass concurrent cycle.
 287       // So code roots might not be scanned, let's scan here.
 288       _cm->concurrent_scan_code_roots(worker_id, rp);
 289     }
 290 
 291     _cm->mark_loop(worker_id, _terminator, rp,
 292                    false, // not cancellable
 293                    _dedup_string);
 294 
 295     assert(_cm->task_queues()->is_empty(), "Should be empty");
 296   }
 297 };
 298 
 299 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
 300   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 301   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 302 
 303   ShenandoahGCPhase phase(root_phase);
 304 
 305   WorkGang* workers = _heap->workers();
 306   uint nworkers = workers->active_workers();
 307 
 308   assert(nworkers <= task_queues()->size(), "Just check");
 309 
 310   ShenandoahAllRootScanner root_proc(root_phase);
 311   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 312   task_queues()->reserve(nworkers);
 313 
 314   if (_heap->has_forwarded_objects()) {
 315     ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc);
 316     workers->run_task(&mark_roots);
 317   } else {
 318     // No need to update references, which means the heap is stable.
 319     // Can save time not walking through forwarding pointers.
 320     ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc);
 321     workers->run_task(&mark_roots);
 322   }
 323 
 324   clear_claim_codecache();
 325 }
 326 
 327 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
 328   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 329   assert(root_phase == ShenandoahPhaseTimings::full_gc_update_roots ||
 330          root_phase == ShenandoahPhaseTimings::degen_gc_update_roots,
 331          "Only for these phases");
 332 
 333   ShenandoahHeap* heap = ShenandoahHeap::heap();
 334   ShenandoahGCPhase phase(root_phase);
 335 
 336   COMPILER2_PRESENT(DerivedPointerTable::clear());
 337 
 338   uint nworkers = heap->workers()->active_workers();
 339 
 340   ShenandoahRootUpdater root_updater(root_phase);
 341   ShenandoahUpdateRootsTask update_roots(&root_updater);
 342   _heap->workers()->run_task(&update_roots);
 343 
 344   COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
 345 }
 346 
 347 class ShenandoahUpdateThreadRootsTask : public AbstractGangTask {
 348 private:
 349   SharedHeap::StrongRootsScope _srs;
 350   ShenandoahPhaseTimings::Phase   _phase;
 351   ShenandoahGCWorkerPhase         _worker_phase;
 352 public:
 353   ShenandoahUpdateThreadRootsTask(bool is_par, ShenandoahPhaseTimings::Phase phase) :
 354     AbstractGangTask("Shenandoah Update Thread Roots"),
 355     _srs(ShenandoahHeap::heap(), true),
 356     _phase(phase),
 357     _worker_phase(phase) {}
 358 
 359   void work(uint worker_id) {
 360     ShenandoahUpdateRefsClosure cl;
 361     ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::ThreadRoots, worker_id);
 362     ResourceMark rm;
 363     Threads::possibly_parallel_oops_do(&cl, NULL, NULL);
 364   }
 365 };
 366 
 367 void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase root_phase) {
 368   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 369 
 370   ShenandoahGCPhase phase(root_phase);
 371 
 372   COMPILER2_PRESENT(DerivedPointerTable::clear());
 373 
 374   WorkGang* workers = _heap->workers();
 375   bool is_par = workers->active_workers() > 1;
 376 
 377   ShenandoahUpdateThreadRootsTask task(is_par, root_phase);
 378   workers->run_task(&task);
 379 
 380   COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
 381 }
 382 
 383 void ShenandoahConcurrentMark::initialize(uint workers) {
 384   _heap = ShenandoahHeap::heap();
 385 
 386   uint num_queues = MAX2(workers, 1U);
 387 
 388   _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
 389 
 390   for (uint i = 0; i < num_queues; ++i) {
 391     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 392     task_queue->initialize();
 393     _task_queues->register_queue(i, task_queue);
 394   }
 395 
 396   JavaThread::satb_mark_queue_set().set_buffer_size(ShenandoahSATBBufferSize);
 397 }
 398 
 399 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
 400   if (claim_codecache()) {
 401     ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
 402     if (!_heap->unload_classes()) {
 403       MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 404       // TODO: We can not honor StringDeduplication here, due to lock ranking
 405       // inversion. So, we may miss some deduplication candidates.
 406       if (_heap->has_forwarded_objects()) {
 407         ShenandoahMarkResolveRefsClosure cl(q, rp);
 408         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 409         CodeCache::blobs_do(&blobs);
 410       } else {
 411         ShenandoahMarkRefsClosure cl(q, rp);
 412         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 413         CodeCache::blobs_do(&blobs);
 414       }
 415     }
 416   }
 417 }
 418 
 419 void ShenandoahConcurrentMark::mark_from_roots() {
 420   WorkGang* workers = _heap->workers();
 421   uint nworkers = workers->active_workers();
 422 
 423   ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
 424 
 425   if (_heap->process_references()) {
 426     ReferenceProcessor* rp = _heap->ref_processor();
 427     rp->set_active_mt_degree(nworkers);
 428 
 429     // enable ("weak") refs discovery
 430     rp->enable_discovery(true /*verify_no_refs*/, true);
 431     rp->setup_policy(_heap->collector_policy()->should_clear_all_soft_refs());
 432   }
 433 
 434   shenandoah_assert_rp_isalive_not_installed();
 435   ShenandoahIsAliveSelector is_alive;
 436   ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 437 
 438   task_queues()->reserve(nworkers);
 439 
 440   {
 441     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 442     ShenandoahConcurrentMarkingTask task(this, &terminator);
 443     workers->run_task(&task);
 444   }
 445 
 446   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
 447   if (!_heap->cancelled_gc()) {
 448     TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
 449   }
 450 
 451   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 452 }
 453 
 454 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
 455   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 456 
 457   uint nworkers = _heap->workers()->active_workers();
 458 
 459   // Finally mark everything else we've got in our queues during the previous steps.
 460   // It does two different things for concurrent vs. mark-compact GC:
 461   // - For concurrent GC, it starts with empty task queues, drains the remaining
 462   //   SATB buffers, and then completes the marking closure.
 463   // - For mark-compact GC, it starts out with the task queues seeded by initial
 464   //   root scan, and completes the closure, thus marking through all live objects
 465   // The implementation is the same, so it's shared here.
 466   {
 467     ShenandoahGCPhase phase(full_gc ?
 468                             ShenandoahPhaseTimings::full_gc_mark_finish_queues :
 469                             ShenandoahPhaseTimings::finish_queues);
 470     task_queues()->reserve(nworkers);
 471 
 472     shenandoah_assert_rp_isalive_not_installed();
 473     ShenandoahIsAliveSelector is_alive;
 474     ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 475 
 476     SharedHeap::StrongRootsScope scope(_heap, true);
 477     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 478     ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
 479     _heap->workers()->run_task(&task);
 480   }
 481 
 482   assert(task_queues()->is_empty(), "Should be empty");
 483 
 484   // Marking is completed, deactivate SATB barrier if it is active
 485   _heap->complete_marking();
 486 
 487   // When we're done marking everything, we process weak references.
 488   // It is not obvious, but reference processing actually calls
 489   // JNIHandle::weak_oops_do() to cleanup JNI and JVMTI weak oops.
 490   if (_heap->process_references()) {
 491     weak_refs_work(full_gc);
 492   }
 493 
 494   // And finally finish class unloading
 495   if (_heap->unload_classes()) {
 496     // We don't mark through weak roots with class unloading cycle,
 497     // so process them here.
 498     weak_roots_work(full_gc);
 499     _heap->unload_classes_and_cleanup_tables(full_gc);
 500   } else if (ShenandoahStringDedup::is_enabled()) {
 501     ShenandoahStringDedup::parallel_cleanup();
 502   }
 503   assert(task_queues()->is_empty(), "Should be empty");
 504   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
 505   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 506 
 507   // Resize Metaspace
 508   MetaspaceGC::compute_new_size();
 509 }
 510 
 511 // Weak Reference Closures
 512 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
 513   uint _worker_id;
 514   ShenandoahTaskTerminator* _terminator;
 515   bool _reset_terminator;
 516 
 517 public:
 518   ShenandoahCMDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 519     _worker_id(worker_id),
 520     _terminator(t),
 521     _reset_terminator(reset_terminator) {
 522   }
 523 
 524   void do_void() {
 525     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 526 
 527     ShenandoahHeap* sh = ShenandoahHeap::heap();
 528     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 529     assert(sh->process_references(), "why else would we be here?");
 530     ReferenceProcessor* rp = sh->ref_processor();
 531 
 532     shenandoah_assert_rp_isalive_installed();
 533 
 534     scm->mark_loop(_worker_id, _terminator, rp,
 535                    false,   // not cancellable
 536                    false);  // do not do strdedup
 537 
 538     if (_reset_terminator) {
 539       _terminator->reset_for_reuse();
 540     }
 541   }
 542 };
 543 
 544 class ShenandoahCMKeepAliveClosure : public OopClosure {
 545 private:
 546   ShenandoahObjToScanQueue* _queue;
 547   ShenandoahHeap* _heap;
 548   ShenandoahMarkingContext* const _mark_context;
 549 
 550   template <class T>
 551   inline void do_oop_nv(T* p) {
 552     ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 553   }
 554 
 555 public:
 556   ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
 557     _queue(q),
 558     _heap(ShenandoahHeap::heap()),
 559     _mark_context(_heap->marking_context()) {}
 560 
 561   void do_oop(narrowOop* p) { do_oop_nv(p); }
 562   void do_oop(oop* p)       { do_oop_nv(p); }
 563 };
 564 
 565 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
 566 private:
 567   ShenandoahObjToScanQueue* _queue;
 568   ShenandoahHeap* _heap;
 569   ShenandoahMarkingContext* const _mark_context;
 570 
 571   template <class T>
 572   inline void do_oop_nv(T* p) {
 573     ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 574   }
 575 
 576 public:
 577   ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 578     _queue(q),
 579     _heap(ShenandoahHeap::heap()),
 580     _mark_context(_heap->marking_context()) {}
 581 
 582   void do_oop(narrowOop* p) { do_oop_nv(p); }
 583   void do_oop(oop* p)       { do_oop_nv(p); }
 584 };
 585 
 586 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
 587 private:
 588   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
 589   ShenandoahTaskTerminator* _terminator;
 590 
 591 public:
 592   ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
 593                              ShenandoahTaskTerminator* t) :
 594     AbstractGangTask("Process reference objects in parallel"),
 595     _proc_task(proc_task),
 596     _terminator(t) {
 597   }
 598 
 599   void work(uint worker_id) {
 600     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 601     ShenandoahHeap* heap = ShenandoahHeap::heap();
 602     ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
 603     if (heap->has_forwarded_objects()) {
 604       ShenandoahForwardedIsAliveClosure is_alive;
 605       ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 606       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 607     } else {
 608       ShenandoahIsAliveClosure is_alive;
 609       ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 610       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 611     }
 612   }
 613 };
 614 
 615 class ShenandoahRefEnqueueTaskProxy : public AbstractGangTask {
 616 private:
 617   AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task;
 618 
 619 public:
 620   ShenandoahRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) :
 621     AbstractGangTask("Enqueue reference objects in parallel"),
 622     _enqueue_task(enqueue_task) {
 623   }
 624 
 625   void work(uint worker_id) {
 626     _enqueue_task.work(worker_id);
 627   }
 628 };
 629 
 630 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
 631 private:
 632   WorkGang* _workers;
 633 
 634 public:
 635   ShenandoahRefProcTaskExecutor(WorkGang* workers) :
 636     _workers(workers) {
 637   }
 638 
 639   // Executes a task using worker threads.
 640   void execute(ProcessTask& task) {
 641     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 642 
 643     // Shortcut execution if task is empty.
 644     // This should be replaced with the generic ReferenceProcessor shortcut,
 645     // see JDK-8181214, JDK-8043575, JDK-6938732.
 646     if (task.is_empty()) {
 647       return;
 648     }
 649 
 650     ShenandoahHeap* heap = ShenandoahHeap::heap();
 651     ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 652     uint nworkers = _workers->active_workers();
 653     cm->task_queues()->reserve(nworkers);
 654 
 655     ShenandoahTaskTerminator terminator(nworkers, cm->task_queues());
 656     ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
 657     _workers->run_task(&proc_task_proxy);
 658   }
 659 
 660   void execute(EnqueueTask& task) {
 661     ShenandoahRefEnqueueTaskProxy enqueue_task_proxy(task);
 662     _workers->run_task(&enqueue_task_proxy);
 663   }
 664 };
 665 
 666 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
 667   assert(_heap->process_references(), "sanity");
 668 
 669   ShenandoahPhaseTimings::Phase phase_root =
 670           full_gc ?
 671           ShenandoahPhaseTimings::full_gc_weakrefs :
 672           ShenandoahPhaseTimings::weakrefs;
 673 
 674   ShenandoahGCPhase phase(phase_root);
 675 
 676   ReferenceProcessor* rp = _heap->ref_processor();
 677   weak_refs_work_doit(full_gc);
 678 
 679   rp->verify_no_references_recorded();
 680   assert(!rp->discovery_enabled(), "Post condition");
 681 
 682 }
 683 
 684 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
 685   ReferenceProcessor* rp = _heap->ref_processor();
 686 
 687   ShenandoahPhaseTimings::Phase phase_process =
 688           full_gc ?
 689           ShenandoahPhaseTimings::full_gc_weakrefs_process :
 690           ShenandoahPhaseTimings::weakrefs_process;
 691 
 692   ShenandoahPhaseTimings::Phase phase_enqueue =
 693           full_gc ?
 694           ShenandoahPhaseTimings::full_gc_weakrefs_enqueue :
 695           ShenandoahPhaseTimings::weakrefs_enqueue;
 696 
 697   shenandoah_assert_rp_isalive_not_installed();
 698   ShenandoahIsAliveSelector is_alive;
 699   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 700 
 701   WorkGang* workers = _heap->workers();
 702   uint nworkers = workers->active_workers();
 703 
 704   rp->setup_policy(_heap->collector_policy()->should_clear_all_soft_refs());
 705   rp->set_active_mt_degree(nworkers);
 706 
 707   assert(task_queues()->is_empty(), "Should be empty");
 708 
 709   // complete_gc and keep_alive closures instantiated here are only needed for
 710   // single-threaded path in RP. They share the queue 0 for tracking work, which
 711   // simplifies implementation. Since RP may decide to call complete_gc several
 712   // times, we need to be able to reuse the terminator.
 713   uint serial_worker_id = 0;
 714   ShenandoahTaskTerminator terminator(1, task_queues());
 715   ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
 716 
 717   ShenandoahRefProcTaskExecutor executor(workers);
 718 
 719   {
 720     ShenandoahGCPhase phase(phase_process);
 721 
 722     if (_heap->has_forwarded_objects()) {
 723       ShenandoahForwardedIsAliveClosure is_alive;
 724       ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
 725       rp->process_discovered_references(&is_alive, &keep_alive,
 726                                         &complete_gc, &executor,
 727                                         NULL, _heap->shenandoah_policy()->tracer()->gc_id());
 728     } else {
 729       ShenandoahIsAliveClosure is_alive;
 730       ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
 731       rp->process_discovered_references(&is_alive, &keep_alive,
 732                                         &complete_gc, &executor,
 733                                         NULL, _heap->shenandoah_policy()->tracer()->gc_id());
 734     }
 735 
 736     assert(task_queues()->is_empty(), "Should be empty");
 737   }
 738 
 739   {
 740     ShenandoahGCPhase phase(phase_enqueue);
 741     rp->enqueue_discovered_references(&executor);
 742   }
 743 }
 744 
 745 class DoNothingClosure: public OopClosure {
 746  public:
 747   void do_oop(oop* p)       {}
 748   void do_oop(narrowOop* p) {}
 749 };
 750 
 751 class ShenandoahWeakUpdateClosure : public OopClosure {
 752 private:
 753   ShenandoahHeap* const _heap;
 754 
 755   template <class T>
 756   inline void do_oop_work(T* p) {
 757     oop o = _heap->maybe_update_with_forwarded(p);
 758     shenandoah_assert_marked_except(p, o, o == NULL);
 759   }
 760 
 761 public:
 762   ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
 763 
 764   void do_oop(narrowOop* p) { do_oop_work(p); }
 765   void do_oop(oop* p)       { do_oop_work(p); }
 766 };
 767 
 768 void ShenandoahConcurrentMark::weak_roots_work(bool full_gc) {
 769   ShenandoahPhaseTimings::Phase phase = full_gc ?
 770                                         ShenandoahPhaseTimings::full_gc_weak_roots :
 771                                         ShenandoahPhaseTimings::weak_roots;
 772   ShenandoahGCPhase root_phase(phase);
 773   ShenandoahGCWorkerPhase worker_phase(phase);
 774 
 775   ShenandoahIsAliveSelector is_alive;
 776   DoNothingClosure cl;
 777   ShenandoahWeakRoots weak_roots(phase);
 778   weak_roots.weak_oops_do(is_alive.is_alive_closure(), &cl, 0);
 779 }
 780 
 781 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
 782 private:
 783   ShenandoahHeap* const _heap;
 784 public:
 785   ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 786   virtual bool should_return() { return _heap->cancelled_gc(); }
 787 };
 788 
 789 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
 790 public:
 791   void do_void() {
 792     ShenandoahHeap* sh = ShenandoahHeap::heap();
 793     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 794     assert(sh->process_references(), "why else would we be here?");
 795     ShenandoahTaskTerminator terminator(1, scm->task_queues());
 796 
 797     ReferenceProcessor* rp = sh->ref_processor();
 798     shenandoah_assert_rp_isalive_installed();
 799 
 800     scm->mark_loop(0, &terminator, rp,
 801                    false, // not cancellable
 802                    false); // do not do strdedup
 803   }
 804 };
 805 
 806 class ShenandoahPrecleanTask : public AbstractGangTask {
 807 private:
 808   ReferenceProcessor* _rp;
 809 
 810 public:
 811   ShenandoahPrecleanTask(ReferenceProcessor* rp) :
 812           AbstractGangTask("Precleaning task"),
 813           _rp(rp) {}
 814 
 815   void work(uint worker_id) {
 816     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 817     ShenandoahParallelWorkerSession worker_session(worker_id);
 818 
 819     ShenandoahHeap* sh = ShenandoahHeap::heap();
 820     assert(!sh->has_forwarded_objects(), "No forwarded objects expected here");
 821 
 822     ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
 823 
 824     ShenandoahCancelledGCYieldClosure yield;
 825     ShenandoahPrecleanCompleteGCClosure complete_gc;
 826 
 827     ShenandoahIsAliveClosure is_alive;
 828     ShenandoahCMKeepAliveClosure keep_alive(q);
 829     ResourceMark rm;
 830     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 831                                        &complete_gc, &yield,
 832                                        NULL, sh->shenandoah_policy()->tracer()->gc_id());
 833   }
 834 };
 835 
 836 void ShenandoahConcurrentMark::preclean_weak_refs() {
 837   // Pre-cleaning weak references before diving into STW makes sense at the
 838   // end of concurrent mark. This will filter out the references which referents
 839   // are alive. Note that ReferenceProcessor already filters out these on reference
 840   // discovery, and the bulk of work is done here. This phase processes leftovers
 841   // that missed the initial filtering, i.e. when referent was marked alive after
 842   // reference was discovered by RP.
 843 
 844   assert(_heap->process_references(), "sanity");
 845 
 846   ReferenceProcessor* rp = _heap->ref_processor();
 847 
 848   assert(task_queues()->is_empty(), "Should be empty");
 849 
 850   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 851 
 852   shenandoah_assert_rp_isalive_not_installed();
 853   ShenandoahIsAliveSelector is_alive;
 854   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 855 
 856   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 857   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 858   // parallel precleans, we can extend this to more threads.
 859   WorkGang* workers = _heap->workers();
 860   uint nworkers = workers->active_workers();
 861   assert(nworkers == 1, "This code uses only a single worker");
 862   task_queues()->reserve(nworkers);
 863 
 864   ShenandoahPrecleanTask task(rp);
 865   workers->run_task(&task);
 866 
 867   assert(task_queues()->is_empty(), "Should be empty");
 868 }
 869 
 870 void ShenandoahConcurrentMark::cancel() {
 871   // Clean up marking stacks.
 872   ShenandoahObjToScanQueueSet* queues = task_queues();
 873   queues->clear();
 874 
 875   // Cancel SATB buffers.
 876   JavaThread::satb_mark_queue_set().abandon_partial_marking();
 877 }
 878 
 879 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
 880   assert(task_queues()->get_reserved() > worker_id, err_msg("No reserved queue for worker id: %d", worker_id));
 881   return _task_queues->queue(worker_id);
 882 }
 883 
 884 template <bool CANCELLABLE>
 885 void ShenandoahConcurrentMark::mark_loop_prework(uint w, ShenandoahTaskTerminator *t, ReferenceProcessor *rp,
 886                                                  bool strdedup) {
 887   ShenandoahObjToScanQueue* q = get_queue(w);
 888 
 889   ShenandoahLiveData* ld = _heap->get_liveness_cache(w);
 890 
 891   // TODO: We can clean up this if we figure out how to do templated oop closures that
 892   // play nice with specialized_oop_iterators.
 893   if (_heap->unload_classes()) {
 894     if (_heap->has_forwarded_objects()) {
 895       if (strdedup) {
 896         ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 897         ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, dq, rp);
 898         mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 899       } else {
 900         ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
 901         mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 902       }
 903     } else {
 904       if (strdedup) {
 905         ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 906         ShenandoahMarkRefsMetadataDedupClosure cl(q, dq, rp);
 907         mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 908       } else {
 909         ShenandoahMarkRefsMetadataClosure cl(q, rp);
 910         mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 911       }
 912     }
 913   } else {
 914     if (_heap->has_forwarded_objects()) {
 915       if (strdedup) {
 916         ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 917         ShenandoahMarkUpdateRefsDedupClosure cl(q, dq, rp);
 918         mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 919       } else {
 920         ShenandoahMarkUpdateRefsClosure cl(q, rp);
 921         mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 922       }
 923     } else {
 924       if (strdedup) {
 925         ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 926         ShenandoahMarkRefsDedupClosure cl(q, dq, rp);
 927         mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 928       } else {
 929         ShenandoahMarkRefsClosure cl(q, rp);
 930         mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 931       }
 932     }
 933   }
 934 
 935   _heap->flush_liveness_cache(w);
 936 }
 937 
 938 template <class T, bool CANCELLABLE>
 939 void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, ShenandoahTaskTerminator *terminator) {
 940   int seed = 17;
 941   uintx stride = ShenandoahMarkLoopStride;
 942 
 943   ShenandoahHeap* heap = ShenandoahHeap::heap();
 944   ShenandoahObjToScanQueueSet* queues = task_queues();
 945   ShenandoahObjToScanQueue* q;
 946   ShenandoahMarkTask t;
 947 
 948   /*
 949    * Process outstanding queues, if any.
 950    *
 951    * There can be more queues than workers. To deal with the imbalance, we claim
 952    * extra queues first. Since marking can push new tasks into the queue associated
 953    * with this worker id, we come back to process this queue in the normal loop.
 954    */
 955   assert(queues->get_reserved() == heap->workers()->active_workers(),
 956     "Need to reserve proper number of queues");
 957 
 958   q = queues->claim_next();
 959   while (q != NULL) {
 960     if (CANCELLABLE && heap->cancelled_gc()) {
 961       return;
 962     }
 963 
 964     for (uint i = 0; i < stride; i++) {
 965       if (q->pop(t)) {
 966         do_task<T>(q, cl, live_data, &t);
 967       } else {
 968         assert(q->is_empty(), "Must be empty");
 969         q = queues->claim_next();
 970         break;
 971       }
 972     }
 973   }
 974 
 975   q = get_queue(worker_id);
 976 
 977   ShenandoahStrDedupQueue *dq = NULL;
 978   if (ShenandoahStringDedup::is_enabled()) {
 979     dq = ShenandoahStringDedup::queue(worker_id);
 980   }
 981 
 982   ShenandoahSATBBufferClosure drain_satb(q, dq);
 983   SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
 984 
 985   /*
 986    * Normal marking loop:
 987    */
 988   while (true) {
 989     if (CANCELLABLE && heap->cancelled_gc()) {
 990       return;
 991     }
 992 
 993     while (satb_mq_set.completed_buffers_num() > 0) {
 994       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 995     }
 996 
 997     uint work = 0;
 998     for (uint i = 0; i < stride; i++) {
 999       if (q->pop(t) ||
1000           queues->steal(worker_id, &seed, t)) {
1001         do_task<T>(q, cl, live_data, &t);
1002         work++;
1003       } else {
1004         break;
1005       }
1006     }
1007 
1008     if (work == 0) {
1009       // No work encountered in current stride, try to terminate.
1010       ShenandoahTerminatorTerminator tt(heap);
1011       if (terminator->offer_termination(&tt)) return;
1012     }
1013   }
1014 }
1015 
1016 bool ShenandoahConcurrentMark::claim_codecache() {
1017   return _claimed_codecache.try_set();
1018 }
1019 
1020 void ShenandoahConcurrentMark::clear_claim_codecache() {
1021   _claimed_codecache.unset();
1022 }