1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/satbMarkQueue.hpp"
28 #include "gc/shared/strongRootsScope.hpp"
29 #include "gc/shared/taskTerminator.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
37 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
38 #include "gc/shenandoah/shenandoahStringDedup.hpp"
39 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
40 #include "gc/shenandoah/shenandoahUtils.hpp"
41 #include "memory/iterator.inline.hpp"
42 #include "memory/resourceArea.hpp"
43 #include "runtime/continuation.hpp"
44 #include "runtime/threads.hpp"
45
46 template <ShenandoahGenerationType GENERATION>
47 class ShenandoahConcurrentMarkingTask : public WorkerTask {
48 private:
49 ShenandoahConcurrentMark* const _cm;
50 TaskTerminator* const _terminator;
51
52 public:
53 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
54 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
55 }
56
57 void work(uint worker_id) {
58 ShenandoahHeap* heap = ShenandoahHeap::heap();
59 ShenandoahConcurrentWorkerSession worker_session(worker_id);
60 ShenandoahSuspendibleThreadSetJoiner stsj;
61 ShenandoahReferenceProcessor* rp = heap->ref_processor();
62 assert(rp != nullptr, "need reference processor");
63 StringDedup::Requests requests;
64 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
65 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
66 &requests);
67 }
68 };
69
70 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
71 private:
72 SATBMarkQueueSet& _satb_qset;
73
74 public:
75 explicit ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset) :
76 _satb_qset(satb_qset) {}
77
78 void do_thread(Thread* thread) override {
79 // Transfer any partial buffer to the qset for completed buffer processing.
80 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
81 }
82 };
83
84 template <ShenandoahGenerationType GENERATION>
85 class ShenandoahFinalMarkingTask : public WorkerTask {
86 private:
87 ShenandoahConcurrentMark* _cm;
88 TaskTerminator* _terminator;
89 bool _dedup_string;
90
91 public:
92 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
93 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
94 }
95
96 void work(uint worker_id) {
97 ShenandoahHeap* heap = ShenandoahHeap::heap();
98
99 ShenandoahParallelWorkerSession worker_session(worker_id);
100 ShenandoahReferenceProcessor* rp = heap->ref_processor();
101 StringDedup::Requests requests;
102
103 // First drain remaining SATB buffers.
104 {
105 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
106
107 ShenandoahSATBBufferClosure<GENERATION> cl(q);
108 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
109 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
110 assert(!heap->has_forwarded_objects(), "Not expected");
111
112 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set);
113 Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
114 }
115 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
116 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
117 &requests);
118 assert(_cm->task_queues()->is_empty(), "Should be empty");
119 }
120 };
121
122 ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
123 ShenandoahMark() {}
124
125 // Mark concurrent roots during concurrent phases
126 template <ShenandoahGenerationType GENERATION>
127 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
128 private:
129 SuspendibleThreadSetJoiner _sts_joiner;
130 ShenandoahConcurrentRootScanner _root_scanner;
131 ShenandoahObjToScanQueueSet* const _queue_set;
132 ShenandoahReferenceProcessor* const _rp;
133
134 public:
135 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
136 ShenandoahReferenceProcessor* rp,
137 ShenandoahPhaseTimings::Phase phase,
138 uint nworkers);
139 void work(uint worker_id);
140 };
141
142 template <ShenandoahGenerationType GENERATION>
143 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
144 ShenandoahReferenceProcessor* rp,
145 ShenandoahPhaseTimings::Phase phase,
146 uint nworkers) :
147 WorkerTask("Shenandoah Concurrent Mark Roots"),
148 _root_scanner(nworkers, phase),
149 _queue_set(qs),
150 _rp(rp) {
151 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
152 }
153
154 template <ShenandoahGenerationType GENERATION>
155 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) {
156 ShenandoahConcurrentWorkerSession worker_session(worker_id);
157 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
158 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp);
159 _root_scanner.roots_do(&cl, worker_id);
160 }
161
162 void ShenandoahConcurrentMark::mark_concurrent_roots() {
163 ShenandoahHeap* const heap = ShenandoahHeap::heap();
164 assert(!heap->has_forwarded_objects(), "Not expected");
165
166 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
167
168 WorkerThreads* workers = heap->workers();
169 ShenandoahReferenceProcessor* rp = heap->ref_processor();
170 task_queues()->reserve(workers->active_workers());
171 ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
172
173 workers->run_task(&task);
174 }
175
176 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
177 private:
178 SATBMarkQueueSet& _qset;
179 public:
180 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
181 HandshakeClosure("Shenandoah Flush SATB"),
182 _qset(qset) {}
183
184 void do_thread(Thread* thread) {
185 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
186 }
187 };
188
189 void ShenandoahConcurrentMark::concurrent_mark() {
190 ShenandoahHeap* const heap = ShenandoahHeap::heap();
191 WorkerThreads* workers = heap->workers();
192 uint nworkers = workers->active_workers();
193 task_queues()->reserve(nworkers);
194
195 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
196 ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
197 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
198 TaskTerminator terminator(nworkers, task_queues());
199 ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator);
200 workers->run_task(&task);
201
202 if (heap->cancelled_gc()) {
203 // GC is cancelled, break out.
204 break;
205 }
206
207 size_t before = qset.completed_buffers_num();
208 {
209 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_mark_satb_flush, true);
210 Handshake::execute(&flush_satb);
211 }
212 size_t after = qset.completed_buffers_num();
213
214 if (before == after) {
215 // No more retries needed, break out.
216 break;
217 }
218 }
219 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
220 }
221
222 void ShenandoahConcurrentMark::finish_mark() {
223 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
224 assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
225 finish_mark_work();
226 assert(task_queues()->is_empty(), "Should be empty");
227 TASKQUEUE_STATS_ONLY(task_queues()->print_and_reset_taskqueue_stats(""));
228
229 ShenandoahHeap* const heap = ShenandoahHeap::heap();
230 heap->set_concurrent_mark_in_progress(false);
231 heap->mark_complete_marking_context();
232
233 end_mark();
234 }
235
236 void ShenandoahConcurrentMark::finish_mark_work() {
237 // Finally mark everything else we've got in our queues during the previous steps.
238 // It does two different things for concurrent vs. mark-compact GC:
239 // - For concurrent GC, it starts with empty task queues, drains the remaining
240 // SATB buffers, and then completes the marking closure.
241 // - For mark-compact GC, it starts out with the task queues seeded by initial
242 // root scan, and completes the closure, thus marking through all live objects
243 // The implementation is the same, so it's shared here.
244 ShenandoahHeap* const heap = ShenandoahHeap::heap();
245 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
246 uint nworkers = heap->workers()->active_workers();
247 task_queues()->reserve(nworkers);
248
249 StrongRootsScope scope(nworkers);
250 TaskTerminator terminator(nworkers, task_queues());
251 ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled());
252 heap->workers()->run_task(&task);
253
254 assert(task_queues()->is_empty(), "Should be empty");
255 }
256
257
258 void ShenandoahConcurrentMark::cancel() {
259 clear();
260 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
261 rp->abandon_partial_discovery();
262 }