1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shared/satbMarkQueue.hpp" 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shared/taskTerminator.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 31 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 33 #include "gc/shenandoah/shenandoahGeneration.hpp" 34 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 35 #include "gc/shenandoah/shenandoahMark.inline.hpp" 36 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 37 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 38 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 39 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 40 #include "gc/shenandoah/shenandoahStringDedup.hpp" 41 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 42 #include "gc/shenandoah/shenandoahUtils.hpp" 43 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 44 #include "memory/iterator.inline.hpp" 45 #include "memory/resourceArea.hpp" 46 47 template <GenerationMode GENERATION> 48 class ShenandoahConcurrentMarkingTask : public WorkerTask { 49 private: 50 ShenandoahConcurrentMark* const _cm; 51 TaskTerminator* const _terminator; 52 53 public: 54 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 55 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 56 } 57 58 void work(uint worker_id) { 59 ShenandoahHeap* heap = ShenandoahHeap::heap(); 60 ShenandoahParallelWorkerSession worker_session(worker_id); 61 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true); 62 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); 63 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 64 assert(rp != NULL, "need reference processor"); 65 StringDedup::Requests requests; 66 _cm->mark_loop(GENERATION, worker_id, _terminator, rp, 67 true /*cancellable*/, 68 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 69 &requests); 70 } 71 }; 72 73 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 74 private: 75 SATBMarkQueueSet& _satb_qset; 76 OopClosure* const _cl; 77 uintx _claim_token; 78 79 public: 80 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 81 _satb_qset(satb_qset), 82 _cl(cl), 83 _claim_token(Threads::thread_claim_token()) {} 84 85 void do_thread(Thread* thread) { 86 if (thread->claim_threads_do(true, _claim_token)) { 87 // Transfer any partial buffer to the qset for completed buffer processing. 88 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 89 if (thread->is_Java_thread()) { 90 if (_cl != NULL) { 91 ResourceMark rm; 92 thread->oops_do(_cl, NULL); 93 } 94 } 95 } 96 } 97 }; 98 99 template<GenerationMode GENERATION> 100 class ShenandoahFinalMarkingTask : public WorkerTask { 101 private: 102 ShenandoahConcurrentMark* _cm; 103 TaskTerminator* _terminator; 104 bool _dedup_string; 105 106 public: 107 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 108 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 109 } 110 111 void work(uint worker_id) { 112 ShenandoahHeap* heap = ShenandoahHeap::heap(); 113 114 ShenandoahParallelWorkerSession worker_session(worker_id); 115 StringDedup::Requests requests; 116 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 117 118 // First drain remaining SATB buffers. 119 { 120 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 121 ShenandoahObjToScanQueue* old = _cm->get_old_queue(worker_id); 122 123 ShenandoahSATBBufferClosure<GENERATION> cl(q, old); 124 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 125 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 126 assert(!heap->has_forwarded_objects(), "Not expected"); 127 128 ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old); 129 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 130 ShenandoahIUBarrier ? &mark_cl : NULL); 131 Threads::threads_do(&tc); 132 } 133 _cm->mark_loop(GENERATION, worker_id, _terminator, rp, 134 false /*not cancellable*/, 135 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 136 &requests); 137 assert(_cm->task_queues()->is_empty(), "Should be empty"); 138 } 139 }; 140 141 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) : 142 ShenandoahMark(generation) {} 143 144 // Mark concurrent roots during concurrent phases 145 template<GenerationMode GENERATION> 146 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 147 private: 148 SuspendibleThreadSetJoiner _sts_joiner; 149 ShenandoahConcurrentRootScanner _root_scanner; 150 ShenandoahObjToScanQueueSet* const _queue_set; 151 ShenandoahObjToScanQueueSet* const _old_queue_set; 152 ShenandoahReferenceProcessor* const _rp; 153 154 public: 155 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 156 ShenandoahObjToScanQueueSet* old, 157 ShenandoahReferenceProcessor* rp, 158 ShenandoahPhaseTimings::Phase phase, 159 uint nworkers); 160 void work(uint worker_id); 161 }; 162 163 template<GenerationMode GENERATION> 164 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 165 ShenandoahObjToScanQueueSet* old, 166 ShenandoahReferenceProcessor* rp, 167 ShenandoahPhaseTimings::Phase phase, 168 uint nworkers) : 169 WorkerTask("Shenandoah Concurrent Mark Roots"), 170 _root_scanner(nworkers, phase), 171 _queue_set(qs), 172 _old_queue_set(old), 173 _rp(rp) { 174 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 175 } 176 177 template<GenerationMode GENERATION> 178 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) { 179 ShenandoahConcurrentWorkerSession worker_session(worker_id); 180 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 181 ShenandoahObjToScanQueue* old = _old_queue_set == NULL ? NULL : _old_queue_set->queue(worker_id); 182 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old); 183 _root_scanner.roots_do(&cl, worker_id); 184 } 185 186 void ShenandoahConcurrentMark::mark_concurrent_roots() { 187 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 188 assert(!heap->has_forwarded_objects(), "Not expected"); 189 190 WorkerThreads* workers = heap->workers(); 191 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 192 _generation->reserve_task_queues(workers->active_workers()); 193 switch (_generation->generation_mode()) { 194 case YOUNG: { 195 ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 196 workers->run_task(&task); 197 break; 198 } 199 case GLOBAL: { 200 assert(old_task_queues() == NULL, "Global mark should not have old gen mark queues."); 201 ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 202 workers->run_task(&task); 203 break; 204 } 205 default: 206 // Intentionally haven't added OLD here. We use a YOUNG generation 207 // cycle to bootstrap concurrent old marking. 208 ShouldNotReachHere(); 209 } 210 } 211 212 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 213 private: 214 SATBMarkQueueSet& _qset; 215 public: 216 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 217 HandshakeClosure("Shenandoah Flush SATB Handshake"), 218 _qset(qset) {} 219 220 void do_thread(Thread* thread) { 221 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 222 } 223 }; 224 225 void ShenandoahConcurrentMark::concurrent_mark() { 226 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 227 WorkerThreads* workers = heap->workers(); 228 uint nworkers = workers->active_workers(); 229 task_queues()->reserve(nworkers); 230 231 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 232 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 233 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 234 switch (_generation->generation_mode()) { 235 case YOUNG: { 236 TaskTerminator terminator(nworkers, task_queues()); 237 ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator); 238 workers->run_task(&task); 239 break; 240 } 241 case OLD: { 242 TaskTerminator terminator(nworkers, task_queues()); 243 ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator); 244 workers->run_task(&task); 245 break; 246 } 247 case GLOBAL: { 248 TaskTerminator terminator(nworkers, task_queues()); 249 ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator); 250 workers->run_task(&task); 251 break; 252 } 253 default: 254 ShouldNotReachHere(); 255 } 256 257 if (heap->cancelled_gc()) { 258 // GC is cancelled, break out. 259 break; 260 } 261 262 size_t before = qset.completed_buffers_num(); 263 Handshake::execute(&flush_satb); 264 size_t after = qset.completed_buffers_num(); 265 266 if (before == after) { 267 // No more retries needed, break out. 268 break; 269 } 270 } 271 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 272 } 273 274 void ShenandoahConcurrentMark::finish_mark() { 275 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 276 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 277 finish_mark_work(); 278 assert(task_queues()->is_empty(), "Should be empty"); 279 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 280 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 281 282 _generation->set_concurrent_mark_in_progress(false); 283 _generation->set_mark_complete(); 284 } 285 286 void ShenandoahConcurrentMark::finish_mark_work() { 287 // Finally mark everything else we've got in our queues during the previous steps. 288 // It does two different things for concurrent vs. mark-compact GC: 289 // - For concurrent GC, it starts with empty task queues, drains the remaining 290 // SATB buffers, and then completes the marking closure. 291 // - For mark-compact GC, it starts out with the task queues seeded by initial 292 // root scan, and completes the closure, thus marking through all live objects 293 // The implementation is the same, so it's shared here. 294 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 295 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 296 uint nworkers = heap->workers()->active_workers(); 297 task_queues()->reserve(nworkers); 298 299 StrongRootsScope scope(nworkers); 300 TaskTerminator terminator(nworkers, task_queues()); 301 302 switch (_generation->generation_mode()) { 303 case YOUNG:{ 304 ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 305 heap->workers()->run_task(&task); 306 break; 307 } 308 case OLD:{ 309 ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 310 heap->workers()->run_task(&task); 311 break; 312 } 313 case GLOBAL:{ 314 ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 315 heap->workers()->run_task(&task); 316 break; 317 } 318 default: 319 ShouldNotReachHere(); 320 } 321 322 323 assert(task_queues()->is_empty(), "Should be empty"); 324 }