1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shared/satbMarkQueue.hpp" 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shared/taskTerminator.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 31 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahMark.inline.hpp" 35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 39 #include "gc/shenandoah/shenandoahStringDedup.hpp" 40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 41 #include "gc/shenandoah/shenandoahUtils.hpp" 42 #include "memory/iterator.inline.hpp" 43 #include "memory/resourceArea.hpp" 44 #include "runtime/continuation.hpp" 45 #include "runtime/threads.hpp" 46 47 class ShenandoahConcurrentMarkingTask : public WorkerTask { 48 private: 49 ShenandoahConcurrentMark* const _cm; 50 TaskTerminator* const _terminator; 51 52 public: 53 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 54 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 55 } 56 57 void work(uint worker_id) { 58 ShenandoahHeap* heap = ShenandoahHeap::heap(); 59 ShenandoahConcurrentWorkerSession worker_session(worker_id); 60 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); 61 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 62 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 63 assert(rp != NULL, "need reference processor"); 64 StringDedup::Requests requests; 65 _cm->mark_loop(worker_id, _terminator, rp, 66 true /*cancellable*/, 67 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 68 &requests); 69 } 70 }; 71 72 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 73 private: 74 SATBMarkQueueSet& _satb_qset; 75 OopClosure* const _cl; 76 uintx _claim_token; 77 78 public: 79 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 80 _satb_qset(satb_qset), 81 _cl(cl), 82 _claim_token(Threads::thread_claim_token()) {} 83 84 void do_thread(Thread* thread) { 85 if (thread->claim_threads_do(true, _claim_token)) { 86 // Transfer any partial buffer to the qset for completed buffer processing. 87 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 88 if (thread->is_Java_thread()) { 89 if (_cl != NULL) { 90 ResourceMark rm; 91 thread->oops_do(_cl, NULL); 92 } 93 } 94 } 95 } 96 }; 97 98 class ShenandoahFinalMarkingTask : public WorkerTask { 99 private: 100 ShenandoahConcurrentMark* _cm; 101 TaskTerminator* _terminator; 102 bool _dedup_string; 103 104 public: 105 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 106 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 107 } 108 109 void work(uint worker_id) { 110 ShenandoahHeap* heap = ShenandoahHeap::heap(); 111 112 ShenandoahParallelWorkerSession worker_session(worker_id); 113 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 114 StringDedup::Requests requests; 115 116 // First drain remaining SATB buffers. 117 { 118 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 119 120 ShenandoahSATBBufferClosure cl(q); 121 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 122 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 123 assert(!heap->has_forwarded_objects(), "Not expected"); 124 125 ShenandoahMarkRefsClosure mark_cl(q, rp); 126 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 127 ShenandoahIUBarrier ? &mark_cl : NULL); 128 Threads::threads_do(&tc); 129 } 130 _cm->mark_loop(worker_id, _terminator, rp, 131 false /*not cancellable*/, 132 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 133 &requests); 134 assert(_cm->task_queues()->is_empty(), "Should be empty"); 135 } 136 }; 137 138 ShenandoahConcurrentMark::ShenandoahConcurrentMark() : 139 ShenandoahMark() {} 140 141 // Mark concurrent roots during concurrent phases 142 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 143 private: 144 SuspendibleThreadSetJoiner _sts_joiner; 145 ShenandoahConcurrentRootScanner _root_scanner; 146 ShenandoahObjToScanQueueSet* const _queue_set; 147 ShenandoahReferenceProcessor* const _rp; 148 149 public: 150 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 151 ShenandoahReferenceProcessor* rp, 152 ShenandoahPhaseTimings::Phase phase, 153 uint nworkers); 154 void work(uint worker_id); 155 }; 156 157 ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 158 ShenandoahReferenceProcessor* rp, 159 ShenandoahPhaseTimings::Phase phase, 160 uint nworkers) : 161 WorkerTask("Shenandoah Concurrent Mark Roots"), 162 _root_scanner(nworkers, phase), 163 _queue_set(qs), 164 _rp(rp) { 165 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 166 } 167 168 void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { 169 ShenandoahConcurrentWorkerSession worker_session(worker_id); 170 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 171 ShenandoahMarkRefsClosure cl(q, _rp); 172 _root_scanner.roots_do(&cl, worker_id); 173 } 174 175 void ShenandoahConcurrentMark::mark_concurrent_roots() { 176 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 177 assert(!heap->has_forwarded_objects(), "Not expected"); 178 179 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 180 181 WorkerThreads* workers = heap->workers(); 182 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 183 task_queues()->reserve(workers->active_workers()); 184 ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 185 186 workers->run_task(&task); 187 } 188 189 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 190 private: 191 SATBMarkQueueSet& _qset; 192 public: 193 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 194 HandshakeClosure("Shenandoah Flush SATB Handshake"), 195 _qset(qset) {} 196 197 void do_thread(Thread* thread) { 198 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 199 } 200 }; 201 202 void ShenandoahConcurrentMark::concurrent_mark() { 203 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 204 WorkerThreads* workers = heap->workers(); 205 uint nworkers = workers->active_workers(); 206 task_queues()->reserve(nworkers); 207 208 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 209 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 210 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 211 TaskTerminator terminator(nworkers, task_queues()); 212 ShenandoahConcurrentMarkingTask task(this, &terminator); 213 workers->run_task(&task); 214 215 if (heap->cancelled_gc()) { 216 // GC is cancelled, break out. 217 break; 218 } 219 220 size_t before = qset.completed_buffers_num(); 221 Handshake::execute(&flush_satb); 222 size_t after = qset.completed_buffers_num(); 223 224 if (before == after) { 225 // No more retries needed, break out. 226 break; 227 } 228 } 229 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 230 } 231 232 void ShenandoahConcurrentMark::finish_mark() { 233 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 234 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 235 finish_mark_work(); 236 assert(task_queues()->is_empty(), "Should be empty"); 237 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 238 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 239 240 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 241 heap->set_concurrent_mark_in_progress(false); 242 heap->mark_complete_marking_context(); 243 244 end_mark(); 245 } 246 247 void ShenandoahConcurrentMark::finish_mark_work() { 248 // Finally mark everything else we've got in our queues during the previous steps. 249 // It does two different things for concurrent vs. mark-compact GC: 250 // - For concurrent GC, it starts with empty task queues, drains the remaining 251 // SATB buffers, and then completes the marking closure. 252 // - For mark-compact GC, it starts out with the task queues seeded by initial 253 // root scan, and completes the closure, thus marking through all live objects 254 // The implementation is the same, so it's shared here. 255 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 256 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 257 uint nworkers = heap->workers()->active_workers(); 258 task_queues()->reserve(nworkers); 259 260 StrongRootsScope scope(nworkers); 261 TaskTerminator terminator(nworkers, task_queues()); 262 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); 263 heap->workers()->run_task(&task); 264 265 assert(task_queues()->is_empty(), "Should be empty"); 266 } 267 268 269 void ShenandoahConcurrentMark::cancel() { 270 clear(); 271 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); 272 rp->abandon_partial_discovery(); 273 }