1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/satbMarkQueue.hpp"
28 #include "gc/shared/strongRootsScope.hpp"
29 #include "gc/shared/taskTerminator.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
39 #include "gc/shenandoah/shenandoahStringDedup.hpp"
40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "memory/iterator.inline.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "runtime/continuation.hpp"
45 #include "runtime/threads.hpp"
46
47 class ShenandoahConcurrentMarkingTask : public WorkerTask {
48 private:
49 ShenandoahConcurrentMark* const _cm;
50 TaskTerminator* const _terminator;
51
52 public:
53 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
54 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
55 }
56
57 void work(uint worker_id) {
58 ShenandoahHeap* heap = ShenandoahHeap::heap();
59 ShenandoahConcurrentWorkerSession worker_session(worker_id);
60 ShenandoahSuspendibleThreadSetJoiner stsj;
61 ShenandoahReferenceProcessor* rp = heap->ref_processor();
62 assert(rp != nullptr, "need reference processor");
63 StringDedup::Requests requests;
64 _cm->mark_loop(worker_id, _terminator, rp,
65 true /*cancellable*/,
66 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
67 &requests);
68 }
69 };
70
71 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
72 private:
73 SATBMarkQueueSet& _satb_qset;
74 OopClosure* const _cl;
75
76 public:
77 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
78 _satb_qset(satb_qset),
79 _cl(cl) {}
80
81 void do_thread(Thread* thread) {
82 // Transfer any partial buffer to the qset for completed buffer processing.
83 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
84 if (thread->is_Java_thread()) {
85 if (_cl != nullptr) {
86 ResourceMark rm;
87 thread->oops_do(_cl, nullptr);
88 }
89 }
90 }
91 };
92
93 class ShenandoahFinalMarkingTask : public WorkerTask {
94 private:
95 ShenandoahConcurrentMark* _cm;
96 TaskTerminator* _terminator;
97 bool _dedup_string;
98
99 public:
100 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
101 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
102 }
103
104 void work(uint worker_id) {
105 ShenandoahHeap* heap = ShenandoahHeap::heap();
106
107 ShenandoahParallelWorkerSession worker_session(worker_id);
108 ShenandoahReferenceProcessor* rp = heap->ref_processor();
109 StringDedup::Requests requests;
110
111 // First drain remaining SATB buffers.
112 {
113 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
114
115 ShenandoahSATBBufferClosure cl(q);
116 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
117 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
118 assert(!heap->has_forwarded_objects(), "Not expected");
119
120 ShenandoahMarkRefsClosure mark_cl(q, rp);
121 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
122 ShenandoahIUBarrier ? &mark_cl : nullptr);
123 Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
124 }
125 _cm->mark_loop(worker_id, _terminator, rp,
126 false /*not cancellable*/,
127 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
128 &requests);
129 assert(_cm->task_queues()->is_empty(), "Should be empty");
130 }
131 };
132
133 ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
134 ShenandoahMark() {}
135
136 // Mark concurrent roots during concurrent phases
137 class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
138 private:
139 SuspendibleThreadSetJoiner _sts_joiner;
140 ShenandoahConcurrentRootScanner _root_scanner;
141 ShenandoahObjToScanQueueSet* const _queue_set;
142 ShenandoahReferenceProcessor* const _rp;
143
144 public:
145 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
146 ShenandoahReferenceProcessor* rp,
147 ShenandoahPhaseTimings::Phase phase,
148 uint nworkers);
149 void work(uint worker_id);
150 };
151
152 ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
153 ShenandoahReferenceProcessor* rp,
154 ShenandoahPhaseTimings::Phase phase,
155 uint nworkers) :
156 WorkerTask("Shenandoah Concurrent Mark Roots"),
157 _root_scanner(nworkers, phase),
158 _queue_set(qs),
159 _rp(rp) {
160 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
161 }
162
163 void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
164 ShenandoahConcurrentWorkerSession worker_session(worker_id);
165 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
166 ShenandoahMarkRefsClosure cl(q, _rp);
167 _root_scanner.roots_do(&cl, worker_id);
168 }
169
170 void ShenandoahConcurrentMark::mark_concurrent_roots() {
171 ShenandoahHeap* const heap = ShenandoahHeap::heap();
172 assert(!heap->has_forwarded_objects(), "Not expected");
173
174 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
175
176 WorkerThreads* workers = heap->workers();
177 ShenandoahReferenceProcessor* rp = heap->ref_processor();
178 task_queues()->reserve(workers->active_workers());
179 ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
180
181 workers->run_task(&task);
182 }
183
184 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
185 private:
186 SATBMarkQueueSet& _qset;
187 public:
188 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
189 HandshakeClosure("Shenandoah Flush SATB Handshake"),
190 _qset(qset) {}
191
192 void do_thread(Thread* thread) {
193 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
194 }
195 };
196
197 void ShenandoahConcurrentMark::concurrent_mark() {
198 ShenandoahHeap* const heap = ShenandoahHeap::heap();
199 WorkerThreads* workers = heap->workers();
200 uint nworkers = workers->active_workers();
201 task_queues()->reserve(nworkers);
202
203 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
204 ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
205 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
206 TaskTerminator terminator(nworkers, task_queues());
207 ShenandoahConcurrentMarkingTask task(this, &terminator);
208 workers->run_task(&task);
209
210 if (heap->cancelled_gc()) {
211 // GC is cancelled, break out.
212 break;
213 }
214
215 size_t before = qset.completed_buffers_num();
216 Handshake::execute(&flush_satb);
217 size_t after = qset.completed_buffers_num();
218
219 if (before == after) {
220 // No more retries needed, break out.
221 break;
222 }
223 }
224 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
225 }
226
227 void ShenandoahConcurrentMark::finish_mark() {
228 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
229 assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
230 finish_mark_work();
231 assert(task_queues()->is_empty(), "Should be empty");
232 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
233 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
234
235 ShenandoahHeap* const heap = ShenandoahHeap::heap();
236 heap->set_concurrent_mark_in_progress(false);
237 heap->mark_complete_marking_context();
238
239 end_mark();
240 }
241
242 void ShenandoahConcurrentMark::finish_mark_work() {
243 // Finally mark everything else we've got in our queues during the previous steps.
244 // It does two different things for concurrent vs. mark-compact GC:
245 // - For concurrent GC, it starts with empty task queues, drains the remaining
246 // SATB buffers, and then completes the marking closure.
247 // - For mark-compact GC, it starts out with the task queues seeded by initial
248 // root scan, and completes the closure, thus marking through all live objects
249 // The implementation is the same, so it's shared here.
250 ShenandoahHeap* const heap = ShenandoahHeap::heap();
251 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
252 uint nworkers = heap->workers()->active_workers();
253 task_queues()->reserve(nworkers);
254
255 StrongRootsScope scope(nworkers);
256 TaskTerminator terminator(nworkers, task_queues());
257 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
258 heap->workers()->run_task(&task);
259
260 assert(task_queues()->is_empty(), "Should be empty");
261 }
262
263
264 void ShenandoahConcurrentMark::cancel() {
265 clear();
266 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
267 rp->abandon_partial_discovery();
268 }