1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shared/satbMarkQueue.hpp" 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shared/taskTerminator.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 31 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahMark.inline.hpp" 35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 39 #include "gc/shenandoah/shenandoahStringDedup.hpp" 40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 41 #include "gc/shenandoah/shenandoahUtils.hpp" 42 #include "memory/iterator.inline.hpp" 43 #include "memory/resourceArea.hpp" 44 #include "runtime/continuation.hpp" 45 #include "runtime/threads.hpp" 46 47 class ShenandoahConcurrentMarkingTask : public WorkerTask { 48 private: 49 ShenandoahConcurrentMark* const _cm; 50 TaskTerminator* const _terminator; 51 52 public: 53 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 54 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 55 } 56 57 void work(uint worker_id) { 58 ShenandoahHeap* heap = ShenandoahHeap::heap(); 59 ShenandoahConcurrentWorkerSession worker_session(worker_id); 60 ShenandoahSuspendibleThreadSetJoiner stsj; 61 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 62 assert(rp != nullptr, "need reference processor"); 63 StringDedup::Requests requests; 64 _cm->mark_loop(worker_id, _terminator, rp, 65 true /*cancellable*/, 66 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 67 &requests); 68 } 69 }; 70 71 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 72 private: 73 SATBMarkQueueSet& _satb_qset; 74 OopClosure* const _cl; 75 uintx _claim_token; 76 77 public: 78 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 79 _satb_qset(satb_qset), 80 _cl(cl) {} 81 82 void do_thread(Thread* thread) { 83 // Transfer any partial buffer to the qset for completed buffer processing. 84 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 85 if (thread->is_Java_thread()) { 86 if (_cl != nullptr) { 87 ResourceMark rm; 88 thread->oops_do(_cl, nullptr); 89 } 90 } 91 } 92 }; 93 94 class ShenandoahFinalMarkingTask : public WorkerTask { 95 private: 96 ShenandoahConcurrentMark* _cm; 97 TaskTerminator* _terminator; 98 bool _dedup_string; 99 100 public: 101 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 102 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 103 } 104 105 void work(uint worker_id) { 106 ShenandoahHeap* heap = ShenandoahHeap::heap(); 107 108 ShenandoahParallelWorkerSession worker_session(worker_id); 109 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 110 StringDedup::Requests requests; 111 112 // First drain remaining SATB buffers. 113 { 114 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 115 116 ShenandoahSATBBufferClosure cl(q); 117 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 118 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 119 assert(!heap->has_forwarded_objects(), "Not expected"); 120 121 ShenandoahMarkRefsClosure mark_cl(q, rp); 122 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 123 ShenandoahIUBarrier ? &mark_cl : nullptr); 124 Threads::possibly_parallel_threads_do(true /* is_par */, &tc); 125 } 126 _cm->mark_loop(worker_id, _terminator, rp, 127 false /*not cancellable*/, 128 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 129 &requests); 130 assert(_cm->task_queues()->is_empty(), "Should be empty"); 131 } 132 }; 133 134 ShenandoahConcurrentMark::ShenandoahConcurrentMark() : 135 ShenandoahMark() {} 136 137 // Mark concurrent roots during concurrent phases 138 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 139 private: 140 SuspendibleThreadSetJoiner _sts_joiner; 141 ShenandoahConcurrentRootScanner _root_scanner; 142 ShenandoahObjToScanQueueSet* const _queue_set; 143 ShenandoahReferenceProcessor* const _rp; 144 145 public: 146 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 147 ShenandoahReferenceProcessor* rp, 148 ShenandoahPhaseTimings::Phase phase, 149 uint nworkers); 150 void work(uint worker_id); 151 }; 152 153 ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 154 ShenandoahReferenceProcessor* rp, 155 ShenandoahPhaseTimings::Phase phase, 156 uint nworkers) : 157 WorkerTask("Shenandoah Concurrent Mark Roots"), 158 _root_scanner(nworkers, phase), 159 _queue_set(qs), 160 _rp(rp) { 161 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 162 } 163 164 void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { 165 ShenandoahConcurrentWorkerSession worker_session(worker_id); 166 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 167 ShenandoahMarkRefsClosure cl(q, _rp); 168 _root_scanner.roots_do(&cl, worker_id); 169 } 170 171 void ShenandoahConcurrentMark::mark_concurrent_roots() { 172 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 173 assert(!heap->has_forwarded_objects(), "Not expected"); 174 175 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 176 177 WorkerThreads* workers = heap->workers(); 178 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 179 task_queues()->reserve(workers->active_workers()); 180 ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 181 182 workers->run_task(&task); 183 } 184 185 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 186 private: 187 SATBMarkQueueSet& _qset; 188 public: 189 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 190 HandshakeClosure("Shenandoah Flush SATB Handshake"), 191 _qset(qset) {} 192 193 void do_thread(Thread* thread) { 194 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 195 } 196 }; 197 198 void ShenandoahConcurrentMark::concurrent_mark() { 199 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 200 WorkerThreads* workers = heap->workers(); 201 uint nworkers = workers->active_workers(); 202 task_queues()->reserve(nworkers); 203 204 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 205 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 206 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 207 TaskTerminator terminator(nworkers, task_queues()); 208 ShenandoahConcurrentMarkingTask task(this, &terminator); 209 workers->run_task(&task); 210 211 if (heap->cancelled_gc()) { 212 // GC is cancelled, break out. 213 break; 214 } 215 216 size_t before = qset.completed_buffers_num(); 217 Handshake::execute(&flush_satb); 218 size_t after = qset.completed_buffers_num(); 219 220 if (before == after) { 221 // No more retries needed, break out. 222 break; 223 } 224 } 225 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 226 } 227 228 void ShenandoahConcurrentMark::finish_mark() { 229 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 230 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 231 finish_mark_work(); 232 assert(task_queues()->is_empty(), "Should be empty"); 233 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 234 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 235 236 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 237 heap->set_concurrent_mark_in_progress(false); 238 heap->mark_complete_marking_context(); 239 240 end_mark(); 241 } 242 243 void ShenandoahConcurrentMark::finish_mark_work() { 244 // Finally mark everything else we've got in our queues during the previous steps. 245 // It does two different things for concurrent vs. mark-compact GC: 246 // - For concurrent GC, it starts with empty task queues, drains the remaining 247 // SATB buffers, and then completes the marking closure. 248 // - For mark-compact GC, it starts out with the task queues seeded by initial 249 // root scan, and completes the closure, thus marking through all live objects 250 // The implementation is the same, so it's shared here. 251 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 252 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 253 uint nworkers = heap->workers()->active_workers(); 254 task_queues()->reserve(nworkers); 255 256 StrongRootsScope scope(nworkers); 257 TaskTerminator terminator(nworkers, task_queues()); 258 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); 259 heap->workers()->run_task(&task); 260 261 assert(task_queues()->is_empty(), "Should be empty"); 262 } 263 264 265 void ShenandoahConcurrentMark::cancel() { 266 clear(); 267 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); 268 rp->abandon_partial_discovery(); 269 }