1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27
28 #include "gc/shared/satbMarkQueue.hpp"
29 #include "gc/shared/strongRootsScope.hpp"
30 #include "gc/shared/taskTerminator.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
34 #include "gc/shenandoah/shenandoahGeneration.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahMark.inline.hpp"
37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
41 #include "gc/shenandoah/shenandoahStringDedup.hpp"
42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
43 #include "gc/shenandoah/shenandoahUtils.hpp"
44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
45 #include "memory/iterator.inline.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "runtime/continuation.hpp"
48 #include "runtime/threads.hpp"
49
50 template <ShenandoahGenerationType GENERATION>
51 class ShenandoahConcurrentMarkingTask : public WorkerTask {
52 private:
53 ShenandoahConcurrentMark* const _cm;
54 TaskTerminator* const _terminator;
55
56 public:
57 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
58 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
59 }
60
61 void work(uint worker_id) {
62 ShenandoahHeap* heap = ShenandoahHeap::heap();
63 ShenandoahParallelWorkerSession worker_session(worker_id);
64 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
65 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
66 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor();
67 assert(rp != nullptr, "need reference processor");
68 StringDedup::Requests requests;
69 _cm->mark_loop(GENERATION, worker_id, _terminator, rp,
70 true /*cancellable*/,
71 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
72 &requests);
73 }
74 };
75
76 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
77 private:
78 SATBMarkQueueSet& _satb_qset;
79 OopClosure* const _cl;
80
81 public:
82 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
83 _satb_qset(satb_qset),
84 _cl(cl) {}
85
86 void do_thread(Thread* thread) {
87 // Transfer any partial buffer to the qset for completed buffer processing.
88 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
89 if (thread->is_Java_thread()) {
90 if (_cl != nullptr) {
91 ResourceMark rm;
92 thread->oops_do(_cl, nullptr);
93 }
94 }
95 }
96 };
97
98 template<ShenandoahGenerationType GENERATION>
99 class ShenandoahFinalMarkingTask : public WorkerTask {
100 private:
101 ShenandoahConcurrentMark* _cm;
102 TaskTerminator* _terminator;
103 bool _dedup_string;
104
105 public:
106 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
107 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
108 }
109
110 void work(uint worker_id) {
111 ShenandoahHeap* heap = ShenandoahHeap::heap();
112
113 ShenandoahParallelWorkerSession worker_session(worker_id);
114 StringDedup::Requests requests;
115 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor();
116
117 // First drain remaining SATB buffers.
118 {
119 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
120 ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
121
122 ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q);
123 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
124 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
125 assert(!heap->has_forwarded_objects(), "Not expected");
126
127 ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q);
128 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
129 ShenandoahIUBarrier ? &mark_cl : nullptr);
130 Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
131 }
132 _cm->mark_loop(GENERATION, worker_id, _terminator, rp,
133 false /*not cancellable*/,
134 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
135 &requests);
136 assert(_cm->task_queues()->is_empty(), "Should be empty");
137 }
138 };
139
140 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
141 ShenandoahMark(generation) {}
142
143 // Mark concurrent roots during concurrent phases
144 template<ShenandoahGenerationType GENERATION>
145 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
146 private:
147 SuspendibleThreadSetJoiner _sts_joiner;
148 ShenandoahConcurrentRootScanner _root_scanner;
149 ShenandoahObjToScanQueueSet* const _queue_set;
150 ShenandoahObjToScanQueueSet* const _old_queue_set;
151 ShenandoahReferenceProcessor* const _rp;
152
153 public:
154 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
155 ShenandoahObjToScanQueueSet* old,
156 ShenandoahReferenceProcessor* rp,
157 ShenandoahPhaseTimings::Phase phase,
158 uint nworkers);
159 void work(uint worker_id);
160 };
161
162 template<ShenandoahGenerationType GENERATION>
163 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
164 ShenandoahObjToScanQueueSet* old,
165 ShenandoahReferenceProcessor* rp,
166 ShenandoahPhaseTimings::Phase phase,
167 uint nworkers) :
168 WorkerTask("Shenandoah Concurrent Mark Roots"),
169 _root_scanner(nworkers, phase),
170 _queue_set(qs),
171 _old_queue_set(old),
172 _rp(rp) {
173 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
174 }
175
176 template<ShenandoahGenerationType GENERATION>
177 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
178 ShenandoahConcurrentWorkerSession worker_session(worker_id);
179 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
180 ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
181 nullptr : _old_queue_set->queue(worker_id);
182 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q);
183 _root_scanner.roots_do(&cl, worker_id);
184 }
185
186 void ShenandoahConcurrentMark::mark_concurrent_roots() {
187 ShenandoahHeap* const heap = ShenandoahHeap::heap();
188 assert(!heap->has_forwarded_objects(), "Not expected");
189
190 WorkerThreads* workers = heap->workers();
191 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
192 _generation->reserve_task_queues(workers->active_workers());
193 switch (_generation->type()) {
194 case YOUNG: {
195 ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp,
196 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
197 workers->run_task(&task);
198 break;
199 }
200 case GLOBAL_GEN: {
201 assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
202 ShenandoahMarkConcurrentRootsTask<GLOBAL_GEN> task(task_queues(), nullptr, rp,
203 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
204 workers->run_task(&task);
205 break;
206 }
207 case GLOBAL_NON_GEN: {
208 assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
209 ShenandoahMarkConcurrentRootsTask<GLOBAL_NON_GEN> task(task_queues(), nullptr, rp,
210 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
211 workers->run_task(&task);
212 break;
213 }
214 case OLD: {
215 // We use a YOUNG generation cycle to bootstrap concurrent old marking.
216 ShouldNotReachHere();
217 break;
218 }
219 default:
220 ShouldNotReachHere();
221 }
222 }
223
224 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
225 private:
226 SATBMarkQueueSet& _qset;
227 public:
228 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
229 HandshakeClosure("Shenandoah Flush SATB Handshake"),
230 _qset(qset) {}
231
232 void do_thread(Thread* thread) {
233 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
234 }
235 };
236
237 void ShenandoahConcurrentMark::concurrent_mark() {
238 ShenandoahHeap* const heap = ShenandoahHeap::heap();
239 WorkerThreads* workers = heap->workers();
240 uint nworkers = workers->active_workers();
241 task_queues()->reserve(nworkers);
242
243 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
244 ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
245 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
246 switch (_generation->type()) {
247 case YOUNG: {
248 // Clear any old/partial local census data before the start of marking.
249 heap->age_census()->reset_local();
250 assert(heap->age_census()->is_clear_local(), "Error");
251 TaskTerminator terminator(nworkers, task_queues());
252 ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
253 workers->run_task(&task);
254 break;
255 }
256 case OLD: {
257 TaskTerminator terminator(nworkers, task_queues());
258 ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator);
259 workers->run_task(&task);
260 break;
261 }
262 case GLOBAL_GEN: {
263 // Clear any old/partial local census data before the start of marking.
264 heap->age_census()->reset_local();
265 assert(heap->age_census()->is_clear_local(), "Error");
266 TaskTerminator terminator(nworkers, task_queues());
267 ShenandoahConcurrentMarkingTask<GLOBAL_GEN> task(this, &terminator);
268 workers->run_task(&task);
269 break;
270 }
271 case GLOBAL_NON_GEN: {
272 TaskTerminator terminator(nworkers, task_queues());
273 ShenandoahConcurrentMarkingTask<GLOBAL_NON_GEN> task(this, &terminator);
274 workers->run_task(&task);
275 break;
276 }
277 default:
278 ShouldNotReachHere();
279 }
280
281 if (heap->cancelled_gc()) {
282 // GC is cancelled, break out.
283 break;
284 }
285
286 size_t before = qset.completed_buffers_num();
287 Handshake::execute(&flush_satb);
288 size_t after = qset.completed_buffers_num();
289
290 if (before == after) {
291 // No more retries needed, break out.
292 break;
293 }
294 }
295 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
296 }
297
298 void ShenandoahConcurrentMark::finish_mark() {
299 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
300 assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
301 finish_mark_work();
302 assert(task_queues()->is_empty(), "Should be empty");
303 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
304 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
305
306 _generation->set_concurrent_mark_in_progress(false);
307 _generation->set_mark_complete();
308
309 end_mark();
310 }
311
312 void ShenandoahConcurrentMark::finish_mark_work() {
313 // Finally mark everything else we've got in our queues during the previous steps.
314 // It does two different things for concurrent vs. mark-compact GC:
315 // - For concurrent GC, it starts with empty task queues, drains the remaining
316 // SATB buffers, and then completes the marking closure.
317 // - For mark-compact GC, it starts out with the task queues seeded by initial
318 // root scan, and completes the closure, thus marking through all live objects
319 // The implementation is the same, so it's shared here.
320 ShenandoahHeap* const heap = ShenandoahHeap::heap();
321 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
322 uint nworkers = heap->workers()->active_workers();
323 task_queues()->reserve(nworkers);
324
325 StrongRootsScope scope(nworkers);
326 TaskTerminator terminator(nworkers, task_queues());
327
328 switch (_generation->type()) {
329 case YOUNG:{
330 ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled());
331 heap->workers()->run_task(&task);
332 break;
333 }
334 case OLD:{
335 ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled());
336 heap->workers()->run_task(&task);
337 break;
338 }
339 case GLOBAL_GEN:{
340 ShenandoahFinalMarkingTask<GLOBAL_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
341 heap->workers()->run_task(&task);
342 break;
343 }
344 case GLOBAL_NON_GEN:{
345 ShenandoahFinalMarkingTask<GLOBAL_NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
346 heap->workers()->run_task(&task);
347 break;
348 }
349 default:
350 ShouldNotReachHere();
351 }
352
353
354 assert(task_queues()->is_empty(), "Should be empty");
355 }