1 /*
  2  * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
  3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 
 28 #include "gc/shared/satbMarkQueue.hpp"
 29 #include "gc/shared/strongRootsScope.hpp"
 30 #include "gc/shared/taskTerminator.hpp"
 31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
 34 #include "gc/shenandoah/shenandoahGeneration.hpp"
 35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 36 #include "gc/shenandoah/shenandoahMark.inline.hpp"
 37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
 38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 41 #include "gc/shenandoah/shenandoahStringDedup.hpp"
 42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
 43 #include "gc/shenandoah/shenandoahUtils.hpp"
 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 45 #include "memory/iterator.inline.hpp"
 46 #include "memory/resourceArea.hpp"
 47 #include "runtime/continuation.hpp"
 48 #include "runtime/threads.hpp"
 49 
 50 template <ShenandoahGenerationType GENERATION>
 51 class ShenandoahConcurrentMarkingTask : public WorkerTask {
 52 private:
 53   ShenandoahConcurrentMark* const _cm;
 54   TaskTerminator* const           _terminator;
 55 
 56 public:
 57   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
 58     WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
 59   }
 60 
 61   void work(uint worker_id) {
 62     ShenandoahHeap* heap = ShenandoahHeap::heap();
 63     ShenandoahParallelWorkerSession worker_session(worker_id);
 64     ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
 65     ShenandoahSuspendibleThreadSetJoiner stsj;
 66     // Do not use active_generation() : we must use the gc_generation() set by
 67     // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
 68     // intervene to update active_generation, so we can't
 69     // shenandoah_assert_generations_reconciled() here.
 70     ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
 71     assert(rp != nullptr, "need reference processor");
 72     StringDedup::Requests requests;
 73     _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
 74                    ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
 75                    &requests);
 76   }
 77 };
 78 
 79 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
 80 private:
 81   SATBMarkQueueSet& _satb_qset;
 82   OopClosure* const _cl;
 83 
 84 public:
 85   ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
 86     _satb_qset(satb_qset),
 87     _cl(cl)  {}
 88 
 89   void do_thread(Thread* thread) {
 90     // Transfer any partial buffer to the qset for completed buffer processing.
 91     _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
 92     if (thread->is_Java_thread()) {
 93       if (_cl != nullptr) {
 94         ResourceMark rm;
 95         thread->oops_do(_cl, nullptr);
 96       }
 97     }
 98   }
 99 };
100 
101 template <ShenandoahGenerationType GENERATION>
102 class ShenandoahFinalMarkingTask : public WorkerTask {
103 private:
104   ShenandoahConcurrentMark* _cm;
105   TaskTerminator*           _terminator;
106   bool                      _dedup_string;
107 
108 public:
109   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
110     WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
111   }
112 
113   void work(uint worker_id) {
114     ShenandoahHeap* heap = ShenandoahHeap::heap();
115 
116     ShenandoahParallelWorkerSession worker_session(worker_id);

117     StringDedup::Requests requests;
118     ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
119     shenandoah_assert_generations_reconciled();
120 
121     // First drain remaining SATB buffers.
122     {
123       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
124       ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
125 
126       ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
127       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
128       while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
129       assert(!heap->has_forwarded_objects(), "Not expected");
130 
131       ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q);
132       ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
133                                                ShenandoahIUBarrier ? &mark_cl : nullptr);
134       Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
135     }
136     _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
137                    _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
138                    &requests);
139     assert(_cm->task_queues()->is_empty(), "Should be empty");
140   }
141 };
142 
143 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
144   ShenandoahMark(generation) {}
145 
146 // Mark concurrent roots during concurrent phases
147 template <ShenandoahGenerationType GENERATION>
148 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
149 private:
150   SuspendibleThreadSetJoiner          _sts_joiner;
151   ShenandoahConcurrentRootScanner     _root_scanner;
152   ShenandoahObjToScanQueueSet* const  _queue_set;
153   ShenandoahObjToScanQueueSet* const  _old_queue_set;
154   ShenandoahReferenceProcessor* const _rp;
155 
156 public:
157   ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
158                                     ShenandoahObjToScanQueueSet* old,
159                                     ShenandoahReferenceProcessor* rp,
160                                     ShenandoahPhaseTimings::Phase phase,
161                                     uint nworkers);
162   void work(uint worker_id);
163 };
164 
165 template <ShenandoahGenerationType GENERATION>
166 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
167                                                                                  ShenandoahObjToScanQueueSet* old,
168                                                                                  ShenandoahReferenceProcessor* rp,
169                                                                                  ShenandoahPhaseTimings::Phase phase,
170                                                                                  uint nworkers) :
171   WorkerTask("Shenandoah Concurrent Mark Roots"),
172   _root_scanner(nworkers, phase),
173   _queue_set(qs),
174   _old_queue_set(old),
175   _rp(rp) {
176   assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
177 }
178 
179 template <ShenandoahGenerationType GENERATION>
180 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
181   ShenandoahConcurrentWorkerSession worker_session(worker_id);
182   ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
183   ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
184           nullptr : _old_queue_set->queue(worker_id);
185   ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
186   _root_scanner.roots_do(&cl, worker_id);
187 }
188 
189 void ShenandoahConcurrentMark::mark_concurrent_roots() {
190   ShenandoahHeap* const heap = ShenandoahHeap::heap();
191   assert(!heap->has_forwarded_objects(), "Not expected");
192 


193   WorkerThreads* workers = heap->workers();
194   ShenandoahReferenceProcessor* rp = _generation->ref_processor();
195   _generation->reserve_task_queues(workers->active_workers());
196   switch (_generation->type()) {
197     case YOUNG: {
198       ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
199                                                     ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
200       workers->run_task(&task);
201       break;
202     }
203     case GLOBAL: {
204       assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
205       ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), nullptr, rp,
206                                                      ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
207       workers->run_task(&task);
208       break;
209     }
210     case NON_GEN: {
211       assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
212       ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), nullptr, rp,
213                                                       ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
214       workers->run_task(&task);
215       break;
216     }
217     case OLD: {
218       // We use a YOUNG generation cycle to bootstrap concurrent old marking.
219       ShouldNotReachHere();
220       break;
221     }
222     default:
223       ShouldNotReachHere();
224   }
225 }
226 
227 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
228 private:
229   SATBMarkQueueSet& _qset;
230 public:
231   ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
232     HandshakeClosure("Shenandoah Flush SATB Handshake"),
233     _qset(qset) {}
234 
235   void do_thread(Thread* thread) {
236     _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
237   }
238 };
239 
240 void ShenandoahConcurrentMark::concurrent_mark() {
241   ShenandoahHeap* const heap = ShenandoahHeap::heap();
242   WorkerThreads* workers = heap->workers();
243   uint nworkers = workers->active_workers();
244   task_queues()->reserve(nworkers);
245 
246   ShenandoahGenerationType gen_type = _generation->type();
247   ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
248   ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
249   for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
250     switch (gen_type) {
251       case YOUNG: {
252         TaskTerminator terminator(nworkers, task_queues());
253         ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
254         workers->run_task(&task);
255         break;
256       }
257       case OLD: {
258         TaskTerminator terminator(nworkers, task_queues());
259         ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
260         workers->run_task(&task);
261         break;
262       }
263       case GLOBAL: {
264         TaskTerminator terminator(nworkers, task_queues());
265         ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator);
266         workers->run_task(&task);
267         break;
268       }
269       case NON_GEN: {
270         TaskTerminator terminator(nworkers, task_queues());
271         ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator);
272         workers->run_task(&task);
273         break;
274       }
275       default:
276         ShouldNotReachHere();
277     }
278 
279     if (heap->cancelled_gc()) {
280       // GC is cancelled, break out.
281       break;
282     }
283 
284     size_t before = qset.completed_buffers_num();
285     Handshake::execute(&flush_satb);
286     size_t after = qset.completed_buffers_num();
287 
288     if (before == after) {
289       // No more retries needed, break out.
290       break;
291     }
292   }
293   assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
294 }
295 
296 void ShenandoahConcurrentMark::finish_mark() {
297   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
298   assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
299   finish_mark_work();
300   assert(task_queues()->is_empty(), "Should be empty");
301   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
302   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
303 
304   _generation->set_concurrent_mark_in_progress(false);
305   _generation->set_mark_complete();

306 
307   end_mark();
308 }
309 
310 void ShenandoahConcurrentMark::finish_mark_work() {
311   // Finally mark everything else we've got in our queues during the previous steps.
312   // It does two different things for concurrent vs. mark-compact GC:
313   // - For concurrent GC, it starts with empty task queues, drains the remaining
314   //   SATB buffers, and then completes the marking closure.
315   // - For mark-compact GC, it starts out with the task queues seeded by initial
316   //   root scan, and completes the closure, thus marking through all live objects
317   // The implementation is the same, so it's shared here.
318   ShenandoahHeap* const heap = ShenandoahHeap::heap();
319   ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
320   uint nworkers = heap->workers()->active_workers();
321   task_queues()->reserve(nworkers);
322 
323   StrongRootsScope scope(nworkers);
324   TaskTerminator terminator(nworkers, task_queues());


325 
326   switch (_generation->type()) {
327     case YOUNG:{
328       ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
329       heap->workers()->run_task(&task);
330       break;
331     }
332     case OLD:{
333       ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
334       heap->workers()->run_task(&task);
335       break;
336     }
337     case GLOBAL:{
338       ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled());
339       heap->workers()->run_task(&task);
340       break;
341     }
342     case NON_GEN:{
343       ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
344       heap->workers()->run_task(&task);
345       break;
346     }
347     default:
348       ShouldNotReachHere();
349   }
350 
351 
352   assert(task_queues()->is_empty(), "Should be empty");



353 }
--- EOF ---