1 /*
  2  * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
  3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 
 28 #include "gc/shared/satbMarkQueue.hpp"
 29 #include "gc/shared/strongRootsScope.hpp"
 30 #include "gc/shared/taskTerminator.hpp"
 31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
 34 #include "gc/shenandoah/shenandoahGeneration.hpp"
 35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 36 #include "gc/shenandoah/shenandoahMark.inline.hpp"
 37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
 38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 41 #include "gc/shenandoah/shenandoahStringDedup.hpp"
 42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 43 #include "gc/shenandoah/shenandoahUtils.hpp"
 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 45 #include "memory/iterator.inline.hpp"
 46 #include "memory/resourceArea.hpp"
 47 #include "runtime/continuation.hpp"
 48 #include "runtime/threads.hpp"
 49 
 50 template <ShenandoahGenerationType GENERATION>
 51 class ShenandoahConcurrentMarkingTask : public WorkerTask {
 52 private:
 53   ShenandoahConcurrentMark* const _cm;
 54   TaskTerminator* const           _terminator;
 55 
 56 public:
 57   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
 58     WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
 59   }
 60 
 61   void work(uint worker_id) {
 62     ShenandoahHeap* heap = ShenandoahHeap::heap();
 63     ShenandoahParallelWorkerSession worker_session(worker_id);
 64     ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
 65     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 66     ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor();
 67     assert(rp != nullptr, "need reference processor");
 68     StringDedup::Requests requests;
 69     _cm->mark_loop(GENERATION, worker_id, _terminator, rp,
 70                    true /*cancellable*/,
 71                    ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
 72                    &requests);
 73   }
 74 };
 75 
 76 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
 77 private:
 78   SATBMarkQueueSet& _satb_qset;
 79   OopClosure* const _cl;
 80 
 81 public:
 82   ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
 83     _satb_qset(satb_qset),
 84     _cl(cl)  {}
 85 
 86   void do_thread(Thread* thread) {
 87     // Transfer any partial buffer to the qset for completed buffer processing.
 88     _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
 89     if (thread->is_Java_thread()) {
 90       if (_cl != nullptr) {
 91         ResourceMark rm;
 92         thread->oops_do(_cl, nullptr);
 93       }
 94     }
 95   }
 96 };
 97 
 98 template<ShenandoahGenerationType GENERATION>
 99 class ShenandoahFinalMarkingTask : public WorkerTask {
100 private:
101   ShenandoahConcurrentMark* _cm;
102   TaskTerminator*           _terminator;
103   bool                      _dedup_string;
104 
105 public:
106   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
107     WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
108   }
109 
110   void work(uint worker_id) {
111     ShenandoahHeap* heap = ShenandoahHeap::heap();
112 
113     ShenandoahParallelWorkerSession worker_session(worker_id);
114     StringDedup::Requests requests;
115     ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor();
116 
117     // First drain remaining SATB buffers.
118     {
119       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
120       ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
121 
122       ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
123       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
124       while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
125       assert(!heap->has_forwarded_objects(), "Not expected");
126 
127       ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q);
128       ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
129                                                ShenandoahIUBarrier ? &mark_cl : nullptr);
130       Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
131     }
132     _cm->mark_loop(GENERATION, worker_id, _terminator, rp,
133                    false /*not cancellable*/,
134                    _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
135                    &requests);
136     assert(_cm->task_queues()->is_empty(), "Should be empty");
137   }
138 };
139 
140 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
141   ShenandoahMark(generation) {}
142 
143 // Mark concurrent roots during concurrent phases
144 template<ShenandoahGenerationType GENERATION>
145 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
146 private:
147   SuspendibleThreadSetJoiner          _sts_joiner;
148   ShenandoahConcurrentRootScanner     _root_scanner;
149   ShenandoahObjToScanQueueSet* const  _queue_set;
150   ShenandoahObjToScanQueueSet* const  _old_queue_set;
151   ShenandoahReferenceProcessor* const _rp;
152 
153 public:
154   ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
155                                     ShenandoahObjToScanQueueSet* old,
156                                     ShenandoahReferenceProcessor* rp,
157                                     ShenandoahPhaseTimings::Phase phase,
158                                     uint nworkers);
159   void work(uint worker_id);
160 };
161 
162 template<ShenandoahGenerationType GENERATION>
163 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
164                                                                                  ShenandoahObjToScanQueueSet* old,
165                                                                                  ShenandoahReferenceProcessor* rp,
166                                                                                  ShenandoahPhaseTimings::Phase phase,
167                                                                                  uint nworkers) :
168   WorkerTask("Shenandoah Concurrent Mark Roots"),
169   _root_scanner(nworkers, phase),
170   _queue_set(qs),
171   _old_queue_set(old),
172   _rp(rp) {
173   assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
174 }
175 
176 template<ShenandoahGenerationType GENERATION>
177 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
178   ShenandoahConcurrentWorkerSession worker_session(worker_id);
179   ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
180   ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
181           nullptr : _old_queue_set->queue(worker_id);
182   ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
183   _root_scanner.roots_do(&cl, worker_id);
184 }
185 
186 void ShenandoahConcurrentMark::mark_concurrent_roots() {
187   ShenandoahHeap* const heap = ShenandoahHeap::heap();
188   assert(!heap->has_forwarded_objects(), "Not expected");
189 
190   WorkerThreads* workers = heap->workers();
191   ShenandoahReferenceProcessor* rp = _generation->ref_processor();
192   _generation->reserve_task_queues(workers->active_workers());
193   switch (_generation->type()) {
194     case YOUNG: {
195       ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
196                                                     ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
197       workers->run_task(&task);
198       break;
199     }
200     case GLOBAL_GEN: {
201       assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
202       ShenandoahMarkConcurrentRootsTask<GLOBAL_GEN> task(task_queues(), nullptr, rp,
203                                                          ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
204       workers->run_task(&task);
205       break;
206     }
207     case GLOBAL_NON_GEN: {
208       assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
209       ShenandoahMarkConcurrentRootsTask<GLOBAL_NON_GEN> task(task_queues(), nullptr, rp,
210                                                          ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
211       workers->run_task(&task);
212       break;
213     }
214     case OLD: {
215       // We use a YOUNG generation cycle to bootstrap concurrent old marking.
216       ShouldNotReachHere();
217       break;
218     }
219     default:
220       ShouldNotReachHere();
221   }
222 }
223 
224 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
225 private:
226   SATBMarkQueueSet& _qset;
227 public:
228   ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
229     HandshakeClosure("Shenandoah Flush SATB Handshake"),
230     _qset(qset) {}
231 
232   void do_thread(Thread* thread) {
233     _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
234   }
235 };
236 
237 void ShenandoahConcurrentMark::concurrent_mark() {
238   ShenandoahHeap* const heap = ShenandoahHeap::heap();
239   WorkerThreads* workers = heap->workers();
240   uint nworkers = workers->active_workers();
241   task_queues()->reserve(nworkers);
242 
243   ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
244   ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
245   for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
246     switch (_generation->type()) {
247       case YOUNG: {
248         // Clear any old/partial local census data before the start of marking.
249         heap->age_census()->reset_local();
250         assert(heap->age_census()->is_clear_local(), "Error");
251         TaskTerminator terminator(nworkers, task_queues());
252         ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
253         workers->run_task(&task);
254         break;
255       }
256       case OLD: {
257         TaskTerminator terminator(nworkers, task_queues());
258         ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
259         workers->run_task(&task);
260         break;
261       }
262       case GLOBAL_GEN: {
263         // Clear any old/partial local census data before the start of marking.
264         heap->age_census()->reset_local();
265         assert(heap->age_census()->is_clear_local(), "Error");
266         TaskTerminator terminator(nworkers, task_queues());
267         ShenandoahConcurrentMarkingTask<GLOBAL_GEN> task(this, &terminator);
268         workers->run_task(&task);
269         break;
270       }
271       case GLOBAL_NON_GEN: {
272         TaskTerminator terminator(nworkers, task_queues());
273         ShenandoahConcurrentMarkingTask<GLOBAL_NON_GEN> task(this, &terminator);
274         workers->run_task(&task);
275         break;
276       }
277       default:
278         ShouldNotReachHere();
279     }
280 
281     if (heap->cancelled_gc()) {
282       // GC is cancelled, break out.
283       break;
284     }
285 
286     size_t before = qset.completed_buffers_num();
287     Handshake::execute(&flush_satb);
288     size_t after = qset.completed_buffers_num();
289 
290     if (before == after) {
291       // No more retries needed, break out.
292       break;
293     }
294   }
295   assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
296 }
297 
298 void ShenandoahConcurrentMark::finish_mark() {
299   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
300   assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
301   finish_mark_work();
302   assert(task_queues()->is_empty(), "Should be empty");
303   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
304   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
305 
306   _generation->set_concurrent_mark_in_progress(false);
307   _generation->set_mark_complete();
308 
309   end_mark();
310 }
311 
312 void ShenandoahConcurrentMark::finish_mark_work() {
313   // Finally mark everything else we've got in our queues during the previous steps.
314   // It does two different things for concurrent vs. mark-compact GC:
315   // - For concurrent GC, it starts with empty task queues, drains the remaining
316   //   SATB buffers, and then completes the marking closure.
317   // - For mark-compact GC, it starts out with the task queues seeded by initial
318   //   root scan, and completes the closure, thus marking through all live objects
319   // The implementation is the same, so it's shared here.
320   ShenandoahHeap* const heap = ShenandoahHeap::heap();
321   ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
322   uint nworkers = heap->workers()->active_workers();
323   task_queues()->reserve(nworkers);
324 
325   StrongRootsScope scope(nworkers);
326   TaskTerminator terminator(nworkers, task_queues());
327 
328   switch (_generation->type()) {
329     case YOUNG:{
330       ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
331       heap->workers()->run_task(&task);
332       break;
333     }
334     case OLD:{
335       ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
336       heap->workers()->run_task(&task);
337       break;
338     }
339     case GLOBAL_GEN:{
340       ShenandoahFinalMarkingTask<GLOBAL_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
341       heap->workers()->run_task(&task);
342       break;
343     }
344     case GLOBAL_NON_GEN:{
345       ShenandoahFinalMarkingTask<GLOBAL_NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
346       heap->workers()->run_task(&task);
347       break;
348     }
349     default:
350       ShouldNotReachHere();
351   }
352 
353 
354   assert(task_queues()->is_empty(), "Should be empty");
355 }