1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 28 #include "gc/shared/satbMarkQueue.hpp" 29 #include "gc/shared/strongRootsScope.hpp" 30 #include "gc/shared/taskTerminator.hpp" 31 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 32 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 33 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 34 #include "gc/shenandoah/shenandoahGeneration.hpp" 35 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 36 #include "gc/shenandoah/shenandoahMark.inline.hpp" 37 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 41 #include "gc/shenandoah/shenandoahStringDedup.hpp" 42 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 43 #include "gc/shenandoah/shenandoahUtils.hpp" 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 45 #include "memory/iterator.inline.hpp" 46 #include "memory/resourceArea.hpp" 47 #include "runtime/continuation.hpp" 48 #include "runtime/threads.hpp" 49 50 template <ShenandoahGenerationType GENERATION> 51 class ShenandoahConcurrentMarkingTask : public WorkerTask { 52 private: 53 ShenandoahConcurrentMark* const _cm; 54 TaskTerminator* const _terminator; 55 56 public: 57 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 58 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 59 } 60 61 void work(uint worker_id) { 62 ShenandoahHeap* heap = ShenandoahHeap::heap(); 63 ShenandoahParallelWorkerSession worker_session(worker_id); 64 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true); 65 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); 66 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 67 assert(rp != nullptr, "need reference processor"); 68 StringDedup::Requests requests; 69 _cm->mark_loop(GENERATION, worker_id, _terminator, rp, 70 true /*cancellable*/, 71 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 72 &requests); 73 } 74 }; 75 76 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 77 private: 78 SATBMarkQueueSet& _satb_qset; 79 OopClosure* const _cl; 80 81 public: 82 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 83 _satb_qset(satb_qset), 84 _cl(cl) {} 85 86 void do_thread(Thread* thread) { 87 // Transfer any partial buffer to the qset for completed buffer processing. 88 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 89 if (thread->is_Java_thread()) { 90 if (_cl != nullptr) { 91 ResourceMark rm; 92 thread->oops_do(_cl, nullptr); 93 } 94 } 95 } 96 }; 97 98 template<ShenandoahGenerationType GENERATION> 99 class ShenandoahFinalMarkingTask : public WorkerTask { 100 private: 101 ShenandoahConcurrentMark* _cm; 102 TaskTerminator* _terminator; 103 bool _dedup_string; 104 105 public: 106 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 107 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 108 } 109 110 void work(uint worker_id) { 111 ShenandoahHeap* heap = ShenandoahHeap::heap(); 112 113 ShenandoahParallelWorkerSession worker_session(worker_id); 114 StringDedup::Requests requests; 115 ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); 116 117 // First drain remaining SATB buffers. 118 { 119 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 120 ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id); 121 122 ShenandoahSATBBufferClosure<GENERATION> cl(q, old_q); 123 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 124 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 125 assert(!heap->has_forwarded_objects(), "Not expected"); 126 127 ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp, old_q); 128 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 129 ShenandoahIUBarrier ? &mark_cl : nullptr); 130 Threads::possibly_parallel_threads_do(true /* is_par */, &tc); 131 } 132 _cm->mark_loop(GENERATION, worker_id, _terminator, rp, 133 false /*not cancellable*/, 134 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 135 &requests); 136 assert(_cm->task_queues()->is_empty(), "Should be empty"); 137 } 138 }; 139 140 ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) : 141 ShenandoahMark(generation) {} 142 143 // Mark concurrent roots during concurrent phases 144 template<ShenandoahGenerationType GENERATION> 145 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 146 private: 147 SuspendibleThreadSetJoiner _sts_joiner; 148 ShenandoahConcurrentRootScanner _root_scanner; 149 ShenandoahObjToScanQueueSet* const _queue_set; 150 ShenandoahObjToScanQueueSet* const _old_queue_set; 151 ShenandoahReferenceProcessor* const _rp; 152 153 public: 154 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 155 ShenandoahObjToScanQueueSet* old, 156 ShenandoahReferenceProcessor* rp, 157 ShenandoahPhaseTimings::Phase phase, 158 uint nworkers); 159 void work(uint worker_id); 160 }; 161 162 template<ShenandoahGenerationType GENERATION> 163 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 164 ShenandoahObjToScanQueueSet* old, 165 ShenandoahReferenceProcessor* rp, 166 ShenandoahPhaseTimings::Phase phase, 167 uint nworkers) : 168 WorkerTask("Shenandoah Concurrent Mark Roots"), 169 _root_scanner(nworkers, phase), 170 _queue_set(qs), 171 _old_queue_set(old), 172 _rp(rp) { 173 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 174 } 175 176 template<ShenandoahGenerationType GENERATION> 177 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) { 178 ShenandoahConcurrentWorkerSession worker_session(worker_id); 179 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 180 ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ? 181 nullptr : _old_queue_set->queue(worker_id); 182 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp, old_q); 183 _root_scanner.roots_do(&cl, worker_id); 184 } 185 186 void ShenandoahConcurrentMark::mark_concurrent_roots() { 187 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 188 assert(!heap->has_forwarded_objects(), "Not expected"); 189 190 WorkerThreads* workers = heap->workers(); 191 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 192 _generation->reserve_task_queues(workers->active_workers()); 193 switch (_generation->type()) { 194 case YOUNG: { 195 ShenandoahMarkConcurrentRootsTask<YOUNG> task(task_queues(), old_task_queues(), rp, 196 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 197 workers->run_task(&task); 198 break; 199 } 200 case GLOBAL_GEN: { 201 assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues"); 202 ShenandoahMarkConcurrentRootsTask<GLOBAL_GEN> task(task_queues(), nullptr, rp, 203 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 204 workers->run_task(&task); 205 break; 206 } 207 case GLOBAL_NON_GEN: { 208 assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues"); 209 ShenandoahMarkConcurrentRootsTask<GLOBAL_NON_GEN> task(task_queues(), nullptr, rp, 210 ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 211 workers->run_task(&task); 212 break; 213 } 214 case OLD: { 215 // We use a YOUNG generation cycle to bootstrap concurrent old marking. 216 ShouldNotReachHere(); 217 break; 218 } 219 default: 220 ShouldNotReachHere(); 221 } 222 } 223 224 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 225 private: 226 SATBMarkQueueSet& _qset; 227 public: 228 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 229 HandshakeClosure("Shenandoah Flush SATB Handshake"), 230 _qset(qset) {} 231 232 void do_thread(Thread* thread) { 233 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 234 } 235 }; 236 237 void ShenandoahConcurrentMark::concurrent_mark() { 238 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 239 WorkerThreads* workers = heap->workers(); 240 uint nworkers = workers->active_workers(); 241 task_queues()->reserve(nworkers); 242 243 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 244 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 245 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 246 switch (_generation->type()) { 247 case YOUNG: { 248 // Clear any old/partial local census data before the start of marking. 249 heap->age_census()->reset_local(); 250 assert(heap->age_census()->is_clear_local(), "Error"); 251 TaskTerminator terminator(nworkers, task_queues()); 252 ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator); 253 workers->run_task(&task); 254 break; 255 } 256 case OLD: { 257 TaskTerminator terminator(nworkers, task_queues()); 258 ShenandoahConcurrentMarkingTask<OLD> task(this, &terminator); 259 workers->run_task(&task); 260 break; 261 } 262 case GLOBAL_GEN: { 263 // Clear any old/partial local census data before the start of marking. 264 heap->age_census()->reset_local(); 265 assert(heap->age_census()->is_clear_local(), "Error"); 266 TaskTerminator terminator(nworkers, task_queues()); 267 ShenandoahConcurrentMarkingTask<GLOBAL_GEN> task(this, &terminator); 268 workers->run_task(&task); 269 break; 270 } 271 case GLOBAL_NON_GEN: { 272 TaskTerminator terminator(nworkers, task_queues()); 273 ShenandoahConcurrentMarkingTask<GLOBAL_NON_GEN> task(this, &terminator); 274 workers->run_task(&task); 275 break; 276 } 277 default: 278 ShouldNotReachHere(); 279 } 280 281 if (heap->cancelled_gc()) { 282 // GC is cancelled, break out. 283 break; 284 } 285 286 size_t before = qset.completed_buffers_num(); 287 Handshake::execute(&flush_satb); 288 size_t after = qset.completed_buffers_num(); 289 290 if (before == after) { 291 // No more retries needed, break out. 292 break; 293 } 294 } 295 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 296 } 297 298 void ShenandoahConcurrentMark::finish_mark() { 299 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 300 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 301 finish_mark_work(); 302 assert(task_queues()->is_empty(), "Should be empty"); 303 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 304 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 305 306 _generation->set_concurrent_mark_in_progress(false); 307 _generation->set_mark_complete(); 308 309 end_mark(); 310 } 311 312 void ShenandoahConcurrentMark::finish_mark_work() { 313 // Finally mark everything else we've got in our queues during the previous steps. 314 // It does two different things for concurrent vs. mark-compact GC: 315 // - For concurrent GC, it starts with empty task queues, drains the remaining 316 // SATB buffers, and then completes the marking closure. 317 // - For mark-compact GC, it starts out with the task queues seeded by initial 318 // root scan, and completes the closure, thus marking through all live objects 319 // The implementation is the same, so it's shared here. 320 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 321 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 322 uint nworkers = heap->workers()->active_workers(); 323 task_queues()->reserve(nworkers); 324 325 StrongRootsScope scope(nworkers); 326 TaskTerminator terminator(nworkers, task_queues()); 327 328 switch (_generation->type()) { 329 case YOUNG:{ 330 ShenandoahFinalMarkingTask<YOUNG> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 331 heap->workers()->run_task(&task); 332 break; 333 } 334 case OLD:{ 335 ShenandoahFinalMarkingTask<OLD> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 336 heap->workers()->run_task(&task); 337 break; 338 } 339 case GLOBAL_GEN:{ 340 ShenandoahFinalMarkingTask<GLOBAL_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 341 heap->workers()->run_task(&task); 342 break; 343 } 344 case GLOBAL_NON_GEN:{ 345 ShenandoahFinalMarkingTask<GLOBAL_NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 346 heap->workers()->run_task(&task); 347 break; 348 } 349 default: 350 ShouldNotReachHere(); 351 } 352 353 354 assert(task_queues()->is_empty(), "Should be empty"); 355 }