1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 28 #include "gc/shared/satbMarkQueue.hpp" 29 #include "gc/shared/strongRootsScope.hpp" 30 #include "gc/shared/taskTerminator.hpp" 31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 32 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 34 #include "gc/shenandoah/shenandoahGeneration.hpp" 35 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 36 #include "gc/shenandoah/shenandoahMark.inline.hpp" 37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 41 #include "gc/shenandoah/shenandoahStringDedup.hpp" 42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 43 #include "gc/shenandoah/shenandoahUtils.hpp" 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 45 #include "memory/iterator.inline.hpp" 46 #include "memory/resourceArea.hpp" 47 #include "runtime/continuation.hpp" 48 #include "runtime/threads.hpp" 49 50 template <ShenandoahGenerationType GENERATION> 51 class ShenandoahConcurrentMarkingTask : public WorkerTask { 52 private: 53 ShenandoahConcurrentMark* const _cm; 54 TaskTerminator* const _terminator; 55 56 public: 57 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 58 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 59 } 60 61 void work(uint worker_id) { 62 ShenandoahHeap* heap = ShenandoahHeap::heap(); 63 ShenandoahParallelWorkerSession worker_session(worker_id); 64 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true); 65 ShenandoahSuspendibleThreadSetJoiner stsj; 66 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 67 assert(rp != nullptr, "need reference processor"); 68 StringDedup::Requests requests; 69 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/, 70 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 71 &requests); 72 } 73 }; 74 75 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 76 private: 77 SATBMarkQueueSet& _satb_qset; 78 OopClosure* const _cl; 79 80 public: 81 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 82 _satb_qset(satb_qset), 83 _cl(cl) {} 84 85 void do_thread(Thread* thread) { 86 // Transfer any partial buffer to the qset for completed buffer processing. 87 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 88 if (thread->is_Java_thread()) { 89 if (_cl != nullptr) { 90 ResourceMark rm; 91 thread->oops_do(_cl, nullptr); 92 } 93 } 94 } 95 }; 96 97 template <ShenandoahGenerationType GENERATION> 98 class ShenandoahFinalMarkingTask : public WorkerTask { 99 private: 100 ShenandoahConcurrentMark* _cm; 101 TaskTerminator* _terminator; 102 bool _dedup_string; 103 104 public: 105 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 106 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 107 } 108 109 void work(uint worker_id) { 110 ShenandoahHeap* heap = ShenandoahHeap::heap(); 111 112 ShenandoahParallelWorkerSession worker_session(worker_id); 113 StringDedup::Requests requests; 114 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 115 116 // First drain remaining SATB buffers. 117 { 118 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 119 ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id); 120 121 ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q); 122 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 123 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 124 assert(!heap->has_forwarded_objects(), "Not expected"); 125 126 ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q); 127 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 128 ShenandoahIUBarrier ? &mark_cl : nullptr); 129 Threads::possibly_parallel_threads_do(true /* is_par */, &tc); 130 } 131 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/, 132 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 133 &requests); 134 assert(_cm->task_queues()->is_empty(), "Should be empty"); 135 } 136 }; 137 138 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) : 139 ShenandoahMark(generation) {} 140 141 // Mark concurrent roots during concurrent phases 142 template <ShenandoahGenerationType GENERATION> 143 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 144 private: 145 SuspendibleThreadSetJoiner _sts_joiner; 146 ShenandoahConcurrentRootScanner _root_scanner; 147 ShenandoahObjToScanQueueSet* const _queue_set; 148 ShenandoahObjToScanQueueSet* const _old_queue_set; 149 ShenandoahReferenceProcessor* const _rp; 150 151 public: 152 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 153 ShenandoahObjToScanQueueSet* old, 154 ShenandoahReferenceProcessor* rp, 155 ShenandoahPhaseTimings::Phase phase, 156 uint nworkers); 157 void work(uint worker_id); 158 }; 159 160 template <ShenandoahGenerationType GENERATION> 161 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 162 ShenandoahObjToScanQueueSet* old, 163 ShenandoahReferenceProcessor* rp, 164 ShenandoahPhaseTimings::Phase phase, 165 uint nworkers) : 166 WorkerTask("Shenandoah Concurrent Mark Roots"), 167 _root_scanner(nworkers, phase), 168 _queue_set(qs), 169 _old_queue_set(old), 170 _rp(rp) { 171 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 172 } 173 174 template <ShenandoahGenerationType GENERATION> 175 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) { 176 ShenandoahConcurrentWorkerSession worker_session(worker_id); 177 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 178 ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ? 179 nullptr : _old_queue_set->queue(worker_id); 180 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q); 181 _root_scanner.roots_do(&cl, worker_id); 182 } 183 184 void ShenandoahConcurrentMark::mark_concurrent_roots() { 185 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 186 assert(!heap->has_forwarded_objects(), "Not expected"); 187 188 WorkerThreads* workers = heap->workers(); 189 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 190 _generation->reserve_task_queues(workers->active_workers()); 191 switch (_generation->type()) { 192 case YOUNG: { 193 ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp, 194 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 195 workers->run_task(&task); 196 break; 197 } 198 case GLOBAL: { 199 assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues"); 200 ShenandoahMarkConcurrentRootsTask<GLOBAL> task(task_queues(), nullptr, rp, 201 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 202 workers->run_task(&task); 203 break; 204 } 205 case NON_GEN: { 206 assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues"); 207 ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), nullptr, rp, 208 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 209 workers->run_task(&task); 210 break; 211 } 212 case OLD: { 213 // We use a YOUNG generation cycle to bootstrap concurrent old marking. 214 ShouldNotReachHere(); 215 break; 216 } 217 default: 218 ShouldNotReachHere(); 219 } 220 } 221 222 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 223 private: 224 SATBMarkQueueSet& _qset; 225 public: 226 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 227 HandshakeClosure("Shenandoah Flush SATB Handshake"), 228 _qset(qset) {} 229 230 void do_thread(Thread* thread) { 231 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 232 } 233 }; 234 235 void ShenandoahConcurrentMark::concurrent_mark() { 236 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 237 WorkerThreads* workers = heap->workers(); 238 uint nworkers = workers->active_workers(); 239 task_queues()->reserve(nworkers); 240 241 ShenandoahGenerationType gen_type = _generation->type(); 242 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 243 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 244 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 245 switch (gen_type) { 246 case YOUNG: { 247 TaskTerminator terminator(nworkers, task_queues()); 248 ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator); 249 workers->run_task(&task); 250 break; 251 } 252 case OLD: { 253 TaskTerminator terminator(nworkers, task_queues()); 254 ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator); 255 workers->run_task(&task); 256 break; 257 } 258 case GLOBAL: { 259 TaskTerminator terminator(nworkers, task_queues()); 260 ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator); 261 workers->run_task(&task); 262 break; 263 } 264 case NON_GEN: { 265 TaskTerminator terminator(nworkers, task_queues()); 266 ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator); 267 workers->run_task(&task); 268 break; 269 } 270 default: 271 ShouldNotReachHere(); 272 } 273 274 if (heap->cancelled_gc()) { 275 // GC is cancelled, break out. 276 break; 277 } 278 279 size_t before = qset.completed_buffers_num(); 280 Handshake::execute(&flush_satb); 281 size_t after = qset.completed_buffers_num(); 282 283 if (before == after) { 284 // No more retries needed, break out. 285 break; 286 } 287 } 288 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 289 } 290 291 void ShenandoahConcurrentMark::finish_mark() { 292 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 293 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 294 finish_mark_work(); 295 assert(task_queues()->is_empty(), "Should be empty"); 296 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 297 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 298 299 _generation->set_concurrent_mark_in_progress(false); 300 _generation->set_mark_complete(); 301 302 end_mark(); 303 } 304 305 void ShenandoahConcurrentMark::finish_mark_work() { 306 // Finally mark everything else we've got in our queues during the previous steps. 307 // It does two different things for concurrent vs. mark-compact GC: 308 // - For concurrent GC, it starts with empty task queues, drains the remaining 309 // SATB buffers, and then completes the marking closure. 310 // - For mark-compact GC, it starts out with the task queues seeded by initial 311 // root scan, and completes the closure, thus marking through all live objects 312 // The implementation is the same, so it's shared here. 313 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 314 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 315 uint nworkers = heap->workers()->active_workers(); 316 task_queues()->reserve(nworkers); 317 318 StrongRootsScope scope(nworkers); 319 TaskTerminator terminator(nworkers, task_queues()); 320 321 switch (_generation->type()) { 322 case YOUNG:{ 323 ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 324 heap->workers()->run_task(&task); 325 break; 326 } 327 case OLD:{ 328 ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 329 heap->workers()->run_task(&task); 330 break; 331 } 332 case GLOBAL:{ 333 ShenandoahFinalMarkingTask<GLOBAL> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 334 heap->workers()->run_task(&task); 335 break; 336 } 337 case NON_GEN:{ 338 ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 339 heap->workers()->run_task(&task); 340 break; 341 } 342 default: 343 ShouldNotReachHere(); 344 } 345 346 347 assert(task_queues()->is_empty(), "Should be empty"); 348 }