1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "classfile/symbolTable.hpp"
  27 #include "classfile/systemDictionary.hpp"
  28 #include "code/codeCache.hpp"
  29 
  30 #include "gc/shared/weakProcessor.hpp"
  31 #include "gc/shared/gcTimer.hpp"
  32 #include "gc/shared/referenceProcessor.hpp"
  33 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  34 
  35 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
  36 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  37 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  38 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  40 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  43 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  44 #include "gc/shenandoah/shenandoahUtils.hpp"
  45 #include "gc/shared/weakProcessor.hpp"
  46 
  47 #include "memory/iterator.inline.hpp"
  48 #include "memory/metaspace.hpp"
  49 #include "memory/resourceArea.hpp"
  50 #include "oops/oop.inline.hpp"
  51 
  52 template<UpdateRefsMode UPDATE_REFS>
  53 class ShenandoahInitMarkRootsClosure : public OopClosure {
  54 private:
  55   ShenandoahObjToScanQueue* _queue;
  56   ShenandoahHeap* _heap;
  57   ShenandoahMarkingContext* const _mark_context;
  58 
  59   template <class T>
  60   inline void do_oop_work(T* p) {
  61     ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context);
  62   }
  63 
  64 public:
  65   ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) :
  66     _queue(q),
  67     _heap(ShenandoahHeap::heap()),
  68     _mark_context(_heap->marking_context()) {};
  69 
  70   void do_oop(narrowOop* p) { do_oop_work(p); }
  71   void do_oop(oop* p)       { do_oop_work(p); }
  72 };
  73 
  74 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
  75   MetadataVisitingOopIterateClosure(rp),
  76   _queue(q),
  77   _heap(ShenandoahHeap::heap()),
  78   _mark_context(_heap->marking_context())
  79 { }
  80 
  81 template<UpdateRefsMode UPDATE_REFS>
  82 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
  83 private:
  84   ShenandoahAllRootScanner* _rp;
  85   bool _process_refs;
  86 public:
  87   ShenandoahInitMarkRootsTask(ShenandoahAllRootScanner* rp, bool process_refs) :
  88     AbstractGangTask("Shenandoah init mark roots task"),
  89     _rp(rp),
  90     _process_refs(process_refs) {
  91   }
  92 
  93   void work(uint worker_id) {
  94     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
  95     ShenandoahParallelWorkerSession worker_session(worker_id);
  96 
  97     ShenandoahHeap* heap = ShenandoahHeap::heap();
  98     ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
  99     assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
 100 
 101     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 102 
 103     ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q);
 104     do_work(heap, &mark_cl, worker_id);
 105   }
 106 
 107 private:
 108   void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
 109     // The rationale for selecting the roots to scan is as follows:
 110     //   a. With unload_classes = true, we only want to scan the actual strong roots from the
 111     //      code cache. This will allow us to identify the dead classes, unload them, *and*
 112     //      invalidate the relevant code cache blobs. This could be only done together with
 113     //      class unloading.
 114     //   b. With unload_classes = false, we have to nominally retain all the references from code
 115     //      cache, because there could be the case of embedded class/oop in the generated code,
 116     //      which we will never visit during mark. Without code cache invalidation, as in (a),
 117     //      we risk executing that code cache blob, and crashing.
 118     if (heap->unload_classes()) {
 119       _rp->strong_roots_do(worker_id, oops);
 120     } else {
 121       _rp->roots_do(worker_id, oops);
 122     }
 123   }
 124 };
 125 
 126 class ShenandoahUpdateRootsTask : public AbstractGangTask {
 127 private:
 128   ShenandoahRootUpdater*  _root_updater;
 129 public:
 130   ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater) :
 131     AbstractGangTask("Shenandoah update roots task"),
 132     _root_updater(root_updater) {
 133   }
 134 
 135   void work(uint worker_id) {
 136     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 137     ShenandoahParallelWorkerSession worker_session(worker_id);
 138 
 139     ShenandoahHeap* heap = ShenandoahHeap::heap();
 140     ShenandoahUpdateRefsClosure cl;
 141     AlwaysTrueClosure always_true;
 142     _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl);
 143   }
 144 };
 145 
 146 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
 147 private:
 148   ShenandoahConcurrentMark* _cm;
 149   ShenandoahTaskTerminator* _terminator;
 150 
 151 public:
 152   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator) :
 153     AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
 154   }
 155 
 156   void work(uint worker_id) {
 157     ShenandoahHeap* heap = ShenandoahHeap::heap();
 158     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 159     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 160     ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 161     ReferenceProcessor* rp;
 162     if (heap->process_references()) {
 163       rp = heap->ref_processor();
 164       shenandoah_assert_rp_isalive_installed();
 165     } else {
 166       rp = NULL;
 167     }
 168 
 169     _cm->concurrent_scan_code_roots(worker_id, rp);
 170     _cm->mark_loop(worker_id, _terminator, rp,
 171                    true, // cancellable
 172                    ShenandoahStringDedup::is_enabled()); // perform string dedup
 173   }
 174 };
 175 
 176 class ShenandoahSATBThreadsClosure : public ThreadClosure {
 177 private:
 178   ShenandoahSATBBufferClosure* _satb_cl;
 179   int _thread_parity;
 180 
 181 public:
 182   ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) :
 183     _satb_cl(satb_cl),
 184     _thread_parity(Threads::thread_claim_parity()) {}
 185 
 186   void do_thread(Thread* thread) {
 187     if (thread->is_Java_thread()) {
 188       if (thread->claim_oops_do(true, _thread_parity)) {
 189         JavaThread* jt = (JavaThread*)thread;
 190         ShenandoahThreadLocalData::satb_mark_queue(jt).apply_closure_and_empty(_satb_cl);
 191       }
 192     } else if (thread->is_VM_thread()) {
 193       if (thread->claim_oops_do(true, _thread_parity)) {
 194         ShenandoahBarrierSet::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
 195       }
 196     }
 197   }
 198 };
 199 
 200 class ShenandoahFinalMarkingTask : public AbstractGangTask {
 201 private:
 202   ShenandoahConcurrentMark* _cm;
 203   ShenandoahTaskTerminator* _terminator;
 204   bool _dedup_string;
 205 
 206 public:
 207   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator, bool dedup_string) :
 208     AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
 209   }
 210 
 211   void work(uint worker_id) {
 212     ShenandoahHeap* heap = ShenandoahHeap::heap();
 213 
 214     ShenandoahParallelWorkerSession worker_session(worker_id);
 215     // First drain remaining SATB buffers.
 216     // Notice that this is not strictly necessary for mark-compact. But since
 217     // it requires a StrongRootsScope around the task, we need to claim the
 218     // threads, and performance-wise it doesn't really matter. Adds about 1ms to
 219     // full-gc.
 220     {
 221       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 222       ShenandoahSATBBufferClosure cl(q);
 223       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 224       while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
 225       ShenandoahSATBThreadsClosure tc(&cl);
 226       Threads::threads_do(&tc);
 227     }
 228 
 229     ReferenceProcessor* rp;
 230     if (heap->process_references()) {
 231       rp = heap->ref_processor();
 232       shenandoah_assert_rp_isalive_installed();
 233     } else {
 234       rp = NULL;
 235     }
 236 
 237     if (heap->is_degenerated_gc_in_progress()) {
 238       // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned,
 239       // let's check here.
 240       _cm->concurrent_scan_code_roots(worker_id, rp);
 241     }
 242 
 243     _cm->mark_loop(worker_id, _terminator, rp,
 244                    false, // not cancellable
 245                    _dedup_string);
 246 
 247     assert(_cm->task_queues()->is_empty(), "Should be empty");
 248   }
 249 };
 250 
 251 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
 252   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 253   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 254 
 255   ShenandoahHeap* heap = ShenandoahHeap::heap();
 256 
 257   ShenandoahGCPhase phase(root_phase);
 258 
 259   WorkGang* workers = heap->workers();
 260   uint nworkers = workers->active_workers();
 261 
 262   assert(nworkers <= task_queues()->size(), "Just check");
 263 
 264   ShenandoahAllRootScanner root_proc(nworkers, root_phase);
 265   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 266   task_queues()->reserve(nworkers);
 267 
 268   if (heap->has_forwarded_objects()) {
 269     ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc, _heap->process_references());
 270     workers->run_task(&mark_roots);
 271   } else {
 272     // No need to update references, which means the heap is stable.
 273     // Can save time not walking through forwarding pointers.
 274     ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc, _heap->process_references());
 275     workers->run_task(&mark_roots);
 276   }
 277 
 278   if (ShenandoahConcurrentScanCodeRoots) {
 279     clear_claim_codecache();
 280   }
 281 }
 282 
 283 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
 284   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 285 
 286   bool update_code_cache = true; // initialize to safer value
 287   switch (root_phase) {
 288     case ShenandoahPhaseTimings::update_roots:
 289     case ShenandoahPhaseTimings::final_update_refs_roots:
 290       update_code_cache = false;
 291       break;
 292     case ShenandoahPhaseTimings::full_gc_roots:
 293     case ShenandoahPhaseTimings::degen_gc_update_roots:
 294       update_code_cache = true;
 295       break;
 296     default:
 297       ShouldNotReachHere();
 298   }
 299 
 300   ShenandoahGCPhase phase(root_phase);
 301 
 302 #if COMPILER2_OR_JVMCI
 303   DerivedPointerTable::clear();
 304 #endif
 305 
 306   uint nworkers = _heap->workers()->active_workers();
 307 
 308   ShenandoahRootUpdater root_updater(nworkers, root_phase, update_code_cache);
 309   ShenandoahUpdateRootsTask update_roots(&root_updater);
 310   _heap->workers()->run_task(&update_roots);
 311 
 312 #if COMPILER2_OR_JVMCI
 313   DerivedPointerTable::update_pointers();
 314 #endif
 315 }
 316 
 317 class ShenandoahUpdateThreadRootsTask : public AbstractGangTask {
 318 private:
 319   ShenandoahThreadRoots           _thread_roots;
 320   ShenandoahPhaseTimings::Phase   _phase;
 321 public:
 322   ShenandoahUpdateThreadRootsTask(bool is_par, ShenandoahPhaseTimings::Phase phase) :
 323     AbstractGangTask("Shenandoah Update Thread Roots"),
 324     _thread_roots(is_par),
 325     _phase(phase) {
 326     ShenandoahHeap::heap()->phase_timings()->record_workers_start(_phase);
 327   }
 328 
 329   ~ShenandoahUpdateThreadRootsTask() {
 330     ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase);
 331   }
 332   void work(uint worker_id) {
 333     ShenandoahUpdateRefsClosure cl;
 334     _thread_roots.oops_do(&cl, NULL, worker_id);
 335   }
 336 };
 337 
 338 void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase root_phase) {
 339   WorkGang* workers = _heap->workers();
 340   bool is_par = workers->active_workers() > 1;
 341 #if COMPILER2_OR_JVMCI
 342   DerivedPointerTable::clear();
 343 #endif
 344   ShenandoahUpdateThreadRootsTask task(is_par, root_phase);
 345   workers->run_task(&task);
 346 #if COMPILER2_OR_JVMCI
 347   DerivedPointerTable::update_pointers();
 348 #endif
 349 }
 350 
 351 void ShenandoahConcurrentMark::initialize(uint workers) {
 352   _heap = ShenandoahHeap::heap();
 353 
 354   uint num_queues = MAX2(workers, 1U);
 355 
 356   _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
 357 
 358   for (uint i = 0; i < num_queues; ++i) {
 359     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 360     task_queue->initialize();
 361     _task_queues->register_queue(i, task_queue);
 362   }
 363 
 364   ShenandoahBarrierSet::satb_mark_queue_set().set_buffer_size(ShenandoahSATBBufferSize);
 365 }
 366 
 367 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
 368   if (ShenandoahConcurrentScanCodeRoots && claim_codecache()) {
 369     ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
 370     if (!_heap->unload_classes()) {
 371       MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 372       // TODO: We can not honor StringDeduplication here, due to lock ranking
 373       // inversion. So, we may miss some deduplication candidates.
 374       if (_heap->has_forwarded_objects()) {
 375         ShenandoahMarkResolveRefsClosure cl(q, rp);
 376         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 377         CodeCache::blobs_do(&blobs);
 378       } else {
 379         ShenandoahMarkRefsClosure cl(q, rp);
 380         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 381         CodeCache::blobs_do(&blobs);
 382       }
 383     }
 384   }
 385 }
 386 
 387 void ShenandoahConcurrentMark::mark_from_roots() {
 388   WorkGang* workers = _heap->workers();
 389   uint nworkers = workers->active_workers();
 390 
 391   ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
 392 
 393   if (_heap->process_references()) {
 394     ReferenceProcessor* rp = _heap->ref_processor();
 395     rp->set_active_mt_degree(nworkers);
 396 
 397     // enable ("weak") refs discovery
 398     rp->enable_discovery(true /*verify_no_refs*/);
 399     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 400   }
 401 
 402   shenandoah_assert_rp_isalive_not_installed();
 403   ShenandoahIsAliveSelector is_alive;
 404   ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 405 
 406   task_queues()->reserve(nworkers);
 407 
 408   {
 409     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::conc_termination);
 410     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 411     ShenandoahConcurrentMarkingTask task(this, &terminator);
 412     workers->run_task(&task);
 413   }
 414 
 415   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
 416 }
 417 
 418 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
 419   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 420 
 421   uint nworkers = _heap->workers()->active_workers();
 422 
 423   // Finally mark everything else we've got in our queues during the previous steps.
 424   // It does two different things for concurrent vs. mark-compact GC:
 425   // - For concurrent GC, it starts with empty task queues, drains the remaining
 426   //   SATB buffers, and then completes the marking closure.
 427   // - For mark-compact GC, it starts out with the task queues seeded by initial
 428   //   root scan, and completes the closure, thus marking through all live objects
 429   // The implementation is the same, so it's shared here.
 430   {
 431     ShenandoahGCPhase phase(full_gc ?
 432                             ShenandoahPhaseTimings::full_gc_mark_finish_queues :
 433                             ShenandoahPhaseTimings::finish_queues);
 434     task_queues()->reserve(nworkers);
 435 
 436     shenandoah_assert_rp_isalive_not_installed();
 437     ShenandoahIsAliveSelector is_alive;
 438     ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 439 
 440     ShenandoahTerminationTracker termination_tracker(full_gc ?
 441                                                      ShenandoahPhaseTimings::full_gc_mark_termination :
 442                                                      ShenandoahPhaseTimings::termination);
 443 
 444     StrongRootsScope scope(nworkers);
 445     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 446     ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
 447     _heap->workers()->run_task(&task);
 448   }
 449 
 450   assert(task_queues()->is_empty(), "Should be empty");
 451 
 452   // When we're done marking everything, we process weak references.
 453   if (_heap->process_references()) {
 454     weak_refs_work(full_gc);
 455   }
 456 
 457   weak_roots_work();
 458 
 459   // And finally finish class unloading
 460   if (_heap->unload_classes()) {
 461     _heap->unload_classes_and_cleanup_tables(full_gc);
 462   } else {
 463     ShenandoahIsAliveSelector alive;
 464     StringTable::unlink(alive.is_alive_closure());
 465   }
 466   if (ShenandoahStringDedup::is_enabled()) {
 467     ShenandoahIsAliveSelector alive;
 468     BoolObjectClosure* is_alive = alive.is_alive_closure();
 469     ShenandoahStringDedup::unlink_or_oops_do(is_alive, NULL, false);
 470   }
 471   assert(task_queues()->is_empty(), "Should be empty");
 472   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
 473   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 474 
 475   // Resize Metaspace
 476   MetaspaceGC::compute_new_size();
 477 }
 478 
 479 // Weak Reference Closures
 480 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
 481   uint _worker_id;
 482   ShenandoahTaskTerminator* _terminator;
 483   bool _reset_terminator;
 484 
 485 public:
 486   ShenandoahCMDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 487     _worker_id(worker_id),
 488     _terminator(t),
 489     _reset_terminator(reset_terminator) {
 490   }
 491 
 492   void do_void() {
 493     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 494 
 495     ShenandoahHeap* sh = ShenandoahHeap::heap();
 496     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 497     assert(sh->process_references(), "why else would we be here?");
 498     ReferenceProcessor* rp = sh->ref_processor();
 499 
 500     shenandoah_assert_rp_isalive_installed();
 501 
 502     scm->mark_loop(_worker_id, _terminator, rp,
 503                    false,   // not cancellable
 504                    false);  // do not do strdedup
 505 
 506     if (_reset_terminator) {
 507       _terminator->reset_for_reuse();
 508     }
 509   }
 510 };
 511 
 512 class ShenandoahCMKeepAliveClosure : public OopClosure {
 513 private:
 514   ShenandoahObjToScanQueue* _queue;
 515   ShenandoahHeap* _heap;
 516   ShenandoahMarkingContext* const _mark_context;
 517 
 518   template <class T>
 519   inline void do_oop_work(T* p) {
 520     ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 521   }
 522 
 523 public:
 524   ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
 525     _queue(q),
 526     _heap(ShenandoahHeap::heap()),
 527     _mark_context(_heap->marking_context()) {}
 528 
 529   void do_oop(narrowOop* p) { do_oop_work(p); }
 530   void do_oop(oop* p)       { do_oop_work(p); }
 531 };
 532 
 533 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
 534 private:
 535   ShenandoahObjToScanQueue* _queue;
 536   ShenandoahHeap* _heap;
 537   ShenandoahMarkingContext* const _mark_context;
 538 
 539   template <class T>
 540   inline void do_oop_work(T* p) {
 541     ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 542   }
 543 
 544 public:
 545   ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 546     _queue(q),
 547     _heap(ShenandoahHeap::heap()),
 548     _mark_context(_heap->marking_context()) {}
 549 
 550   void do_oop(narrowOop* p) { do_oop_work(p); }
 551   void do_oop(oop* p)       { do_oop_work(p); }
 552 };
 553 
 554 class ShenandoahWeakUpdateClosure : public OopClosure {
 555 private:
 556   ShenandoahHeap* const _heap;
 557 
 558   template <class T>
 559   inline void do_oop_work(T* p) {
 560     oop o = _heap->maybe_update_with_forwarded(p);
 561     shenandoah_assert_marked_except(p, o, o == NULL);
 562   }
 563 
 564 public:
 565   ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
 566 
 567   void do_oop(narrowOop* p) { do_oop_work(p); }
 568   void do_oop(oop* p)       { do_oop_work(p); }
 569 };
 570 
 571 class ShenandoahWeakAssertNotForwardedClosure : public OopClosure {
 572 private:
 573   template <class T>
 574   inline void do_oop_work(T* p) {
 575 #ifdef ASSERT
 576     T o = RawAccess<>::oop_load(p);
 577     if (!CompressedOops::is_null(o)) {
 578       oop obj = CompressedOops::decode_not_null(o);
 579       shenandoah_assert_not_forwarded(p, obj);
 580     }
 581 #endif
 582   }
 583 
 584 public:
 585   ShenandoahWeakAssertNotForwardedClosure() {}
 586 
 587   void do_oop(narrowOop* p) { do_oop_work(p); }
 588   void do_oop(oop* p)       { do_oop_work(p); }
 589 };
 590 
 591 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
 592 private:
 593   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
 594   ShenandoahTaskTerminator* _terminator;
 595 
 596 public:
 597   ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
 598                              ShenandoahTaskTerminator* t) :
 599     AbstractGangTask("Process reference objects in parallel"),
 600     _proc_task(proc_task),
 601     _terminator(t) {
 602   }
 603 
 604   void work(uint worker_id) {
 605     ResourceMark rm;
 606     HandleMark hm;
 607     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 608     ShenandoahHeap* heap = ShenandoahHeap::heap();
 609     ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
 610     if (heap->has_forwarded_objects()) {
 611       ShenandoahForwardedIsAliveClosure is_alive;
 612       ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 613       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 614     } else {
 615       ShenandoahIsAliveClosure is_alive;
 616       ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 617       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 618     }
 619   }
 620 };
 621 
 622 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
 623 private:
 624   WorkGang* _workers;
 625 
 626 public:
 627   ShenandoahRefProcTaskExecutor(WorkGang* workers) :
 628     _workers(workers) {
 629   }
 630 
 631   // Executes a task using worker threads.
 632   void execute(ProcessTask& task, uint ergo_workers) {
 633     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 634 
 635     ShenandoahHeap* heap = ShenandoahHeap::heap();
 636     ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 637     ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
 638                                           ergo_workers,
 639                                           /* do_check = */ false);
 640     uint nworkers = _workers->active_workers();
 641     cm->task_queues()->reserve(nworkers);
 642     ShenandoahTaskTerminator terminator(nworkers, cm->task_queues());
 643     ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
 644     _workers->run_task(&proc_task_proxy);
 645   }
 646 };
 647 
 648 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
 649   assert(_heap->process_references(), "sanity");
 650 
 651   ShenandoahPhaseTimings::Phase phase_root =
 652           full_gc ?
 653           ShenandoahPhaseTimings::full_gc_weakrefs :
 654           ShenandoahPhaseTimings::weakrefs;
 655 
 656   ShenandoahGCPhase phase(phase_root);
 657 
 658   ReferenceProcessor* rp = _heap->ref_processor();
 659 
 660   // NOTE: We cannot shortcut on has_discovered_references() here, because
 661   // we will miss marking JNI Weak refs then, see implementation in
 662   // ReferenceProcessor::process_discovered_references.
 663   weak_refs_work_doit(full_gc);
 664 
 665   rp->verify_no_references_recorded();
 666   assert(!rp->discovery_enabled(), "Post condition");
 667 
 668 }
 669 
 670 // Process leftover weak oops: update them, if needed or assert they do not
 671 // need updating otherwise.
 672 // Weak processor API requires us to visit the oops, even if we are not doing
 673 // anything to them.
 674 void ShenandoahConcurrentMark::weak_roots_work() {
 675   OopClosure* keep_alive = &do_nothing_cl;
 676 #ifdef ASSERT
 677   ShenandoahWeakAssertNotForwardedClosure verify_cl;
 678   keep_alive = &verify_cl;
 679 #endif
 680   ShenandoahIsAliveClosure is_alive;
 681   WeakProcessor::weak_oops_do(&is_alive, keep_alive);
 682 }
 683 
 684 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
 685   ReferenceProcessor* rp = _heap->ref_processor();
 686 
 687   ShenandoahPhaseTimings::Phase phase_process =
 688           full_gc ?
 689           ShenandoahPhaseTimings::full_gc_weakrefs_process :
 690           ShenandoahPhaseTimings::weakrefs_process;
 691 
 692   ShenandoahPhaseTimings::Phase phase_process_termination =
 693           full_gc ?
 694           ShenandoahPhaseTimings::full_gc_weakrefs_termination :
 695           ShenandoahPhaseTimings::weakrefs_termination;
 696 
 697   shenandoah_assert_rp_isalive_not_installed();
 698   ShenandoahIsAliveSelector is_alive;
 699   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 700 
 701   WorkGang* workers = _heap->workers();
 702   uint nworkers = workers->active_workers();
 703 
 704   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 705   rp->set_active_mt_degree(nworkers);
 706 
 707   assert(task_queues()->is_empty(), "Should be empty");
 708 
 709   // complete_gc and keep_alive closures instantiated here are only needed for
 710   // single-threaded path in RP. They share the queue 0 for tracking work, which
 711   // simplifies implementation. Since RP may decide to call complete_gc several
 712   // times, we need to be able to reuse the terminator.
 713   uint serial_worker_id = 0;
 714   ShenandoahTaskTerminator terminator(1, task_queues());
 715   ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
 716 
 717   ShenandoahRefProcTaskExecutor executor(workers);
 718 
 719   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
 720 
 721   {
 722     ShenandoahGCPhase phase(phase_process);
 723     ShenandoahTerminationTracker phase_term(phase_process_termination);
 724 
 725     if (_heap->has_forwarded_objects()) {
 726       ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
 727       rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 728                                         &complete_gc, &executor,
 729                                         &pt);
 730 
 731     } else {
 732       ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
 733       rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 734                                         &complete_gc, &executor,
 735                                         &pt);
 736 
 737     }
 738 
 739     pt.print_all_references();
 740 
 741     assert(task_queues()->is_empty(), "Should be empty");
 742   }
 743 }
 744 
 745 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
 746 private:
 747   ShenandoahHeap* const _heap;
 748 public:
 749   ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 750   virtual bool should_return() { return _heap->cancelled_gc(); }
 751 };
 752 
 753 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
 754 public:
 755   void do_void() {
 756     ShenandoahHeap* sh = ShenandoahHeap::heap();
 757     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 758     assert(sh->process_references(), "why else would we be here?");
 759     ShenandoahTaskTerminator terminator(1, scm->task_queues());
 760 
 761     ReferenceProcessor* rp = sh->ref_processor();
 762     shenandoah_assert_rp_isalive_installed();
 763 
 764     scm->mark_loop(0, &terminator, rp,
 765                    false, // not cancellable
 766                    false); // do not do strdedup
 767   }
 768 };
 769 
 770 class ShenandoahPrecleanKeepAliveUpdateClosure : public OopClosure {
 771 private:
 772   ShenandoahObjToScanQueue* _queue;
 773   ShenandoahHeap* _heap;
 774   ShenandoahMarkingContext* const _mark_context;
 775 
 776   template <class T>
 777   inline void do_oop_work(T* p) {
 778     ShenandoahConcurrentMark::mark_through_ref<T, CONCURRENT, NO_DEDUP>(p, _heap, _queue, _mark_context);
 779   }
 780 
 781 public:
 782   ShenandoahPrecleanKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 783     _queue(q),
 784     _heap(ShenandoahHeap::heap()),
 785     _mark_context(_heap->marking_context()) {}
 786 
 787   void do_oop(narrowOop* p) { do_oop_work(p); }
 788   void do_oop(oop* p)       { do_oop_work(p); }
 789 };
 790 
 791 class ShenandoahPrecleanTask : public AbstractGangTask {
 792 private:
 793   ReferenceProcessor* _rp;
 794 
 795 public:
 796   ShenandoahPrecleanTask(ReferenceProcessor* rp) :
 797           AbstractGangTask("Precleaning task"),
 798           _rp(rp) {}
 799 
 800   void work(uint worker_id) {
 801     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 802     ShenandoahParallelWorkerSession worker_session(worker_id);
 803 
 804     ShenandoahHeap* sh = ShenandoahHeap::heap();
 805 
 806     ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
 807 
 808     ShenandoahCancelledGCYieldClosure yield;
 809     ShenandoahPrecleanCompleteGCClosure complete_gc;
 810 
 811     if (sh->has_forwarded_objects()) {
 812       ShenandoahForwardedIsAliveClosure is_alive;
 813       ShenandoahPrecleanKeepAliveUpdateClosure keep_alive(q);
 814       ResourceMark rm;
 815       _rp->preclean_discovered_references(&is_alive, &keep_alive,
 816                                           &complete_gc, &yield,
 817                                           NULL);
 818     } else {
 819       ShenandoahIsAliveClosure is_alive;
 820       ShenandoahCMKeepAliveClosure keep_alive(q);
 821       ResourceMark rm;
 822       _rp->preclean_discovered_references(&is_alive, &keep_alive,
 823                                           &complete_gc, &yield,
 824                                           NULL);
 825     }
 826   }
 827 };
 828 
 829 void ShenandoahConcurrentMark::preclean_weak_refs() {
 830   // Pre-cleaning weak references before diving into STW makes sense at the
 831   // end of concurrent mark. This will filter out the references which referents
 832   // are alive. Note that ReferenceProcessor already filters out these on reference
 833   // discovery, and the bulk of work is done here. This phase processes leftovers
 834   // that missed the initial filtering, i.e. when referent was marked alive after
 835   // reference was discovered by RP.
 836 
 837   assert(_heap->process_references(), "sanity");
 838 
 839   // Shortcut if no references were discovered to avoid winding up threads.
 840   ReferenceProcessor* rp = _heap->ref_processor();
 841   if (!rp->has_discovered_references()) {
 842     return;
 843   }
 844 
 845   assert(task_queues()->is_empty(), "Should be empty");
 846 
 847   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 848 
 849   shenandoah_assert_rp_isalive_not_installed();
 850   ShenandoahIsAliveSelector is_alive;
 851   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 852 
 853   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 854   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 855   // parallel precleans, we can extend this to more threads.
 856   WorkGang* workers = _heap->workers();
 857   uint nworkers = workers->active_workers();
 858   assert(nworkers == 1, "This code uses only a single worker");
 859   task_queues()->reserve(nworkers);
 860 
 861   ShenandoahPrecleanTask task(rp);
 862   workers->run_task(&task);
 863 
 864   assert(task_queues()->is_empty(), "Should be empty");
 865 }
 866 
 867 void ShenandoahConcurrentMark::cancel() {
 868   // Clean up marking stacks.
 869   ShenandoahObjToScanQueueSet* queues = task_queues();
 870   queues->clear();
 871 
 872   // Cancel SATB buffers.
 873   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
 874 }
 875 
 876 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
 877   assert(task_queues()->get_reserved() > worker_id, "No reserved queue for worker id: %d", worker_id);
 878   return _task_queues->queue(worker_id);
 879 }
 880 
 881 template <bool CANCELLABLE>
 882 void ShenandoahConcurrentMark::mark_loop_prework(uint w, ShenandoahTaskTerminator *t, ReferenceProcessor *rp,
 883                                                  bool strdedup) {
 884   ShenandoahObjToScanQueue* q = get_queue(w);
 885 
 886   jushort* ld = _heap->get_liveness_cache(w);
 887 
 888   // TODO: We can clean up this if we figure out how to do templated oop closures that
 889   // play nice with specialized_oop_iterators.
 890   if (_heap->unload_classes()) {
 891     if (_heap->has_forwarded_objects()) {
 892       if (strdedup) {
 893         ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp);
 894         mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 895       } else {
 896         ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
 897         mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 898       }
 899     } else {
 900       if (strdedup) {
 901         ShenandoahMarkRefsMetadataDedupClosure cl(q, rp);
 902         mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 903       } else {
 904         ShenandoahMarkRefsMetadataClosure cl(q, rp);
 905         mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 906       }
 907     }
 908   } else {
 909     if (_heap->has_forwarded_objects()) {
 910       if (strdedup) {
 911         ShenandoahMarkUpdateRefsDedupClosure cl(q, rp);
 912         mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 913       } else {
 914         ShenandoahMarkUpdateRefsClosure cl(q, rp);
 915         mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 916       }
 917     } else {
 918       if (strdedup) {
 919         ShenandoahMarkRefsDedupClosure cl(q, rp);
 920         mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 921       } else {
 922         ShenandoahMarkRefsClosure cl(q, rp);
 923         mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 924       }
 925     }
 926   }
 927 
 928   _heap->flush_liveness_cache(w);
 929 }
 930 
 931 template <class T, bool CANCELLABLE>
 932 void ShenandoahConcurrentMark::mark_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator *terminator) {
 933   int seed = 17;
 934   uintx stride = ShenandoahMarkLoopStride;
 935 
 936   ShenandoahHeap* heap = ShenandoahHeap::heap();
 937   ShenandoahObjToScanQueueSet* queues = task_queues();
 938   ShenandoahObjToScanQueue* q;
 939   ShenandoahMarkTask t;
 940 
 941   /*
 942    * Process outstanding queues, if any.
 943    *
 944    * There can be more queues than workers. To deal with the imbalance, we claim
 945    * extra queues first. Since marking can push new tasks into the queue associated
 946    * with this worker id, we come back to process this queue in the normal loop.
 947    */
 948   assert(queues->get_reserved() == heap->workers()->active_workers(),
 949          "Need to reserve proper number of queues: reserved: %u, active: %u", queues->get_reserved(), heap->workers()->active_workers());
 950 
 951   q = queues->claim_next();
 952   while (q != NULL) {
 953     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 954       return;
 955     }
 956 
 957     for (uint i = 0; i < stride; i++) {
 958       if (q->pop(t)) {
 959         do_task<T>(q, cl, live_data, &t);
 960       } else {
 961         assert(q->is_empty(), "Must be empty");
 962         q = queues->claim_next();
 963         break;
 964       }
 965     }
 966   }
 967   q = get_queue(worker_id);
 968 
 969   ShenandoahSATBBufferClosure drain_satb(q);
 970   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 971 
 972   /*
 973    * Normal marking loop:
 974    */
 975   while (true) {
 976     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 977       return;
 978     }
 979 
 980     while (satb_mq_set.completed_buffers_num() > 0) {
 981       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 982     }
 983 
 984     uint work = 0;
 985     for (uint i = 0; i < stride; i++) {
 986       if (q->pop(t) ||
 987           queues->steal(worker_id, &seed, t)) {
 988         do_task<T>(q, cl, live_data, &t);
 989         work++;
 990       } else {
 991         break;
 992       }
 993     }
 994 
 995     if (work == 0) {
 996       // No work encountered in current stride, try to terminate.
 997       // Need to leave the STS here otherwise it might block safepoints.
 998       ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE && ShenandoahSuspendibleWorkers);
 999       ShenandoahTerminationTimingsTracker term_tracker(worker_id);
1000       ShenandoahTerminatorTerminator tt(heap);
1001       if (terminator->offer_termination(&tt)) return;
1002     }
1003   }
1004 }
1005 
1006 bool ShenandoahConcurrentMark::claim_codecache() {
1007   assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
1008   return _claimed_codecache.try_set();
1009 }
1010 
1011 void ShenandoahConcurrentMark::clear_claim_codecache() {
1012   assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
1013   _claimed_codecache.unset();
1014 }