1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27
28 #include "gc/shared/satbMarkQueue.hpp"
29 #include "gc/shared/strongRootsScope.hpp"
30 #include "gc/shared/taskTerminator.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
34 #include "gc/shenandoah/shenandoahGeneration.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahMark.inline.hpp"
37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
39 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
40 #include "gc/shenandoah/shenandoahStringDedup.hpp"
41 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
42 #include "gc/shenandoah/shenandoahUtils.hpp"
43 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
44 #include "memory/iterator.inline.hpp"
45 #include "memory/resourceArea.hpp"
46 #include "runtime/continuation.hpp"
47 #include "runtime/threads.hpp"
48
49 template <ShenandoahGenerationType GENERATION>
50 class ShenandoahConcurrentMarkingTask : public WorkerTask {
51 private:
52 ShenandoahConcurrentMark* const _cm;
53 TaskTerminator* const _terminator;
54
55 public:
56 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
57 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
58 }
59
60 void work(uint worker_id) {
61 ShenandoahHeap* heap = ShenandoahHeap::heap();
62 ShenandoahConcurrentWorkerSession worker_session(worker_id);
63 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
64 ShenandoahSuspendibleThreadSetJoiner stsj;
65 // Do not use active_generation() : we must use the gc_generation() set by
66 // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
67 // intervene to update active_generation, so we can't
68 // shenandoah_assert_generations_reconciled() here.
69 ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
70 assert(rp != nullptr, "need reference processor");
71 StringDedup::Requests requests;
72 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
73 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
74 &requests);
75 }
76 };
77
78 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
79 private:
80 SATBMarkQueueSet& _satb_qset;
81
82 public:
83 explicit ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset) :
84 _satb_qset(satb_qset) {}
85
86 void do_thread(Thread* thread) override {
87 // Transfer any partial buffer to the qset for completed buffer processing.
88 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
89 }
90 };
91
92 template <ShenandoahGenerationType GENERATION>
93 class ShenandoahFinalMarkingTask : public WorkerTask {
94 private:
95 ShenandoahConcurrentMark* _cm;
96 TaskTerminator* _terminator;
97 bool _dedup_string;
98
99 public:
100 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
101 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
102 }
103
104 void work(uint worker_id) {
105 ShenandoahHeap* heap = ShenandoahHeap::heap();
106
107 ShenandoahParallelWorkerSession worker_session(worker_id);
108 StringDedup::Requests requests;
109 ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
110 shenandoah_assert_generations_reconciled();
111
112 // First drain remaining SATB buffers.
113 {
114 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
115 ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
116
117 ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
118 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
119 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
120 assert(!heap->has_forwarded_objects(), "Not expected");
121
122 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set);
123 Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
124 }
125 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
126 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
127 &requests);
128 assert(_cm->task_queues()->is_empty(), "Should be empty");
129 }
130 };
131
132 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
133 ShenandoahMark(generation) {}
134
135 // Mark concurrent roots during concurrent phases
136 template <ShenandoahGenerationType GENERATION>
137 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
138 private:
139 SuspendibleThreadSetJoiner _sts_joiner;
140 ShenandoahConcurrentRootScanner _root_scanner;
141 ShenandoahObjToScanQueueSet* const _queue_set;
142 ShenandoahObjToScanQueueSet* const _old_queue_set;
143 ShenandoahReferenceProcessor* const _rp;
144
145 public:
146 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
147 ShenandoahObjToScanQueueSet* old,
148 ShenandoahReferenceProcessor* rp,
149 ShenandoahPhaseTimings::Phase phase,
150 uint nworkers);
151 void work(uint worker_id);
152 };
153
154 template <ShenandoahGenerationType GENERATION>
155 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
156 ShenandoahObjToScanQueueSet* old,
157 ShenandoahReferenceProcessor* rp,
158 ShenandoahPhaseTimings::Phase phase,
159 uint nworkers) :
160 WorkerTask("Shenandoah Concurrent Mark Roots"),
161 _root_scanner(nworkers, phase),
162 _queue_set(qs),
163 _old_queue_set(old),
164 _rp(rp) {
165 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
166 }
167
168 template <ShenandoahGenerationType GENERATION>
169 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
170 ShenandoahConcurrentWorkerSession worker_session(worker_id);
171 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
172 ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
173 nullptr : _old_queue_set->queue(worker_id);
174 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
175 _root_scanner.roots_do(&cl, worker_id);
176 }
177
178 void ShenandoahConcurrentMark::mark_concurrent_roots() {
179 ShenandoahHeap* const heap = ShenandoahHeap::heap();
180 assert(!heap->has_forwarded_objects(), "Not expected");
181
182 WorkerThreads* workers = heap->workers();
183 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
184 _generation->reserve_task_queues(workers->active_workers());
185 switch (_generation->type()) {
186 case YOUNG: {
187 ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
188 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
189 workers->run_task(&task);
190 break;
191 }
192 case GLOBAL: {
193 assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
194 ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), nullptr, rp,
195 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
196 workers->run_task(&task);
197 break;
198 }
199 case NON_GEN: {
200 assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
201 ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), nullptr, rp,
202 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
203 workers->run_task(&task);
204 break;
205 }
206 case OLD: {
207 // We use a YOUNG generation cycle to bootstrap concurrent old marking.
208 ShouldNotReachHere();
209 break;
210 }
211 default:
212 ShouldNotReachHere();
213 }
214 }
215
216 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
217 private:
218 SATBMarkQueueSet& _qset;
219 public:
220 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
221 HandshakeClosure("Shenandoah Flush SATB"),
222 _qset(qset) {}
223
224 void do_thread(Thread* thread) {
225 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
226 }
227 };
228
229 void ShenandoahConcurrentMark::concurrent_mark() {
230 ShenandoahHeap* const heap = ShenandoahHeap::heap();
231 WorkerThreads* workers = heap->workers();
232 uint nworkers = workers->active_workers();
233 task_queues()->reserve(nworkers);
234
235 ShenandoahGenerationType gen_type = _generation->type();
236 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
237 ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
238 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
239 switch (gen_type) {
240 case YOUNG: {
241 TaskTerminator terminator(nworkers, task_queues());
242 ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
243 workers->run_task(&task);
244 break;
245 }
246 case OLD: {
247 TaskTerminator terminator(nworkers, task_queues());
248 ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
249 workers->run_task(&task);
250 break;
251 }
252 case GLOBAL: {
253 TaskTerminator terminator(nworkers, task_queues());
254 ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator);
255 workers->run_task(&task);
256 break;
257 }
258 case NON_GEN: {
259 TaskTerminator terminator(nworkers, task_queues());
260 ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator);
261 workers->run_task(&task);
262 break;
263 }
264 default:
265 ShouldNotReachHere();
266 }
267
268 if (heap->cancelled_gc()) {
269 // GC is cancelled, break out.
270 break;
271 }
272
273 size_t before = qset.completed_buffers_num();
274 {
275 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_mark_satb_flush, true);
276 Handshake::execute(&flush_satb);
277 }
278 size_t after = qset.completed_buffers_num();
279
280 if (before == after) {
281 // No more retries needed, break out.
282 break;
283 }
284 }
285 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
286 }
287
288 void ShenandoahConcurrentMark::finish_mark() {
289 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
290 assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
291 finish_mark_work();
292 assert(task_queues()->is_empty(), "Should be empty");
293 TASKQUEUE_STATS_ONLY(task_queues()->print_and_reset_taskqueue_stats(""));
294
295 _generation->set_concurrent_mark_in_progress(false);
296 _generation->set_mark_complete();
297
298 end_mark();
299 }
300
301 void ShenandoahConcurrentMark::finish_mark_work() {
302 // Finally mark everything else we've got in our queues during the previous steps.
303 // It does two different things for concurrent vs. mark-compact GC:
304 // - For concurrent GC, it starts with empty task queues, drains the remaining
305 // SATB buffers, and then completes the marking closure.
306 // - For mark-compact GC, it starts out with the task queues seeded by initial
307 // root scan, and completes the closure, thus marking through all live objects
308 // The implementation is the same, so it's shared here.
309 ShenandoahHeap* const heap = ShenandoahHeap::heap();
310 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
311 uint nworkers = heap->workers()->active_workers();
312 task_queues()->reserve(nworkers);
313
314 StrongRootsScope scope(nworkers);
315 TaskTerminator terminator(nworkers, task_queues());
316
317 switch (_generation->type()) {
318 case YOUNG:{
319 ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
320 heap->workers()->run_task(&task);
321 break;
322 }
323 case OLD:{
324 ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
325 heap->workers()->run_task(&task);
326 break;
327 }
328 case GLOBAL:{
329 ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled());
330 heap->workers()->run_task(&task);
331 break;
332 }
333 case NON_GEN:{
334 ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
335 heap->workers()->run_task(&task);
336 break;
337 }
338 default:
339 ShouldNotReachHere();
340 }
341
342
343 assert(task_queues()->is_empty(), "Should be empty");
344 }