1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shared/satbMarkQueue.hpp" 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shared/taskTerminator.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" 31 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahMark.inline.hpp" 35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 39 #include "gc/shenandoah/shenandoahStringDedup.hpp" 40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 41 #include "gc/shenandoah/shenandoahUtils.hpp" 42 #include "memory/iterator.inline.hpp" 43 #include "memory/resourceArea.hpp" 44 #include "runtime/continuation.hpp" 45 #include "runtime/threads.hpp" 46 47 template <ShenandoahGenerationType GENERATION> 48 class ShenandoahConcurrentMarkingTask : public WorkerTask { 49 private: 50 ShenandoahConcurrentMark* const _cm; 51 TaskTerminator* const _terminator; 52 53 public: 54 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) : 55 WorkerTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) { 56 } 57 58 void work(uint worker_id) { 59 ShenandoahHeap* heap = ShenandoahHeap::heap(); 60 ShenandoahConcurrentWorkerSession worker_session(worker_id); 61 ShenandoahSuspendibleThreadSetJoiner stsj; 62 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 63 assert(rp != nullptr, "need reference processor"); 64 StringDedup::Requests requests; 65 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/, 66 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, 67 &requests); 68 } 69 }; 70 71 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { 72 private: 73 SATBMarkQueueSet& _satb_qset; 74 OopClosure* const _cl; 75 76 public: 77 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) : 78 _satb_qset(satb_qset), 79 _cl(cl) {} 80 81 void do_thread(Thread* thread) { 82 // Transfer any partial buffer to the qset for completed buffer processing. 83 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 84 if (thread->is_Java_thread()) { 85 if (_cl != nullptr) { 86 ResourceMark rm; 87 thread->oops_do(_cl, nullptr); 88 } 89 } 90 } 91 }; 92 93 template <ShenandoahGenerationType GENERATION> 94 class ShenandoahFinalMarkingTask : public WorkerTask { 95 private: 96 ShenandoahConcurrentMark* _cm; 97 TaskTerminator* _terminator; 98 bool _dedup_string; 99 100 public: 101 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) : 102 WorkerTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) { 103 } 104 105 void work(uint worker_id) { 106 ShenandoahHeap* heap = ShenandoahHeap::heap(); 107 108 ShenandoahParallelWorkerSession worker_session(worker_id); 109 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 110 StringDedup::Requests requests; 111 112 // First drain remaining SATB buffers. 113 { 114 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 115 116 ShenandoahSATBBufferClosure<GENERATION> cl(q); 117 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); 118 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} 119 assert(!heap->has_forwarded_objects(), "Not expected"); 120 121 ShenandoahMarkRefsClosure<GENERATION> mark_cl(q, rp); 122 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, 123 ShenandoahIUBarrier ? &mark_cl : nullptr); 124 Threads::possibly_parallel_threads_do(true /* is_par */, &tc); 125 } 126 _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/, 127 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, 128 &requests); 129 assert(_cm->task_queues()->is_empty(), "Should be empty"); 130 } 131 }; 132 133 ShenandoahConcurrentMark::ShenandoahConcurrentMark() : 134 ShenandoahMark() {} 135 136 // Mark concurrent roots during concurrent phases 137 template <ShenandoahGenerationType GENERATION> 138 class ShenandoahMarkConcurrentRootsTask : public WorkerTask { 139 private: 140 SuspendibleThreadSetJoiner _sts_joiner; 141 ShenandoahConcurrentRootScanner _root_scanner; 142 ShenandoahObjToScanQueueSet* const _queue_set; 143 ShenandoahReferenceProcessor* const _rp; 144 145 public: 146 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 147 ShenandoahReferenceProcessor* rp, 148 ShenandoahPhaseTimings::Phase phase, 149 uint nworkers); 150 void work(uint worker_id); 151 }; 152 153 template <ShenandoahGenerationType GENERATION> 154 ShenandoahMarkConcurrentRootsTask<GENERATION>::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, 155 ShenandoahReferenceProcessor* rp, 156 ShenandoahPhaseTimings::Phase phase, 157 uint nworkers) : 158 WorkerTask("Shenandoah Concurrent Mark Roots"), 159 _root_scanner(nworkers, phase), 160 _queue_set(qs), 161 _rp(rp) { 162 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); 163 } 164 165 template <ShenandoahGenerationType GENERATION> 166 void ShenandoahMarkConcurrentRootsTask<GENERATION>::work(uint worker_id) { 167 ShenandoahConcurrentWorkerSession worker_session(worker_id); 168 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); 169 ShenandoahMarkRefsClosure<GENERATION> cl(q, _rp); 170 _root_scanner.roots_do(&cl, worker_id); 171 } 172 173 void ShenandoahConcurrentMark::mark_concurrent_roots() { 174 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 175 assert(!heap->has_forwarded_objects(), "Not expected"); 176 177 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 178 179 WorkerThreads* workers = heap->workers(); 180 ShenandoahReferenceProcessor* rp = heap->ref_processor(); 181 task_queues()->reserve(workers->active_workers()); 182 ShenandoahMarkConcurrentRootsTask<NON_GEN> task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); 183 184 workers->run_task(&task); 185 } 186 187 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { 188 private: 189 SATBMarkQueueSet& _qset; 190 public: 191 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) : 192 HandshakeClosure("Shenandoah Flush SATB Handshake"), 193 _qset(qset) {} 194 195 void do_thread(Thread* thread) { 196 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 197 } 198 }; 199 200 void ShenandoahConcurrentMark::concurrent_mark() { 201 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 202 WorkerThreads* workers = heap->workers(); 203 uint nworkers = workers->active_workers(); 204 task_queues()->reserve(nworkers); 205 206 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); 207 ShenandoahFlushSATBHandshakeClosure flush_satb(qset); 208 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { 209 TaskTerminator terminator(nworkers, task_queues()); 210 ShenandoahConcurrentMarkingTask<NON_GEN> task(this, &terminator); 211 workers->run_task(&task); 212 213 if (heap->cancelled_gc()) { 214 // GC is cancelled, break out. 215 break; 216 } 217 218 size_t before = qset.completed_buffers_num(); 219 Handshake::execute(&flush_satb); 220 size_t after = qset.completed_buffers_num(); 221 222 if (before == after) { 223 // No more retries needed, break out. 224 break; 225 } 226 } 227 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); 228 } 229 230 void ShenandoahConcurrentMark::finish_mark() { 231 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 232 assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); 233 finish_mark_work(); 234 assert(task_queues()->is_empty(), "Should be empty"); 235 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); 236 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); 237 238 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 239 heap->set_concurrent_mark_in_progress(false); 240 heap->mark_complete_marking_context(); 241 242 end_mark(); 243 } 244 245 void ShenandoahConcurrentMark::finish_mark_work() { 246 // Finally mark everything else we've got in our queues during the previous steps. 247 // It does two different things for concurrent vs. mark-compact GC: 248 // - For concurrent GC, it starts with empty task queues, drains the remaining 249 // SATB buffers, and then completes the marking closure. 250 // - For mark-compact GC, it starts out with the task queues seeded by initial 251 // root scan, and completes the closure, thus marking through all live objects 252 // The implementation is the same, so it's shared here. 253 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 254 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); 255 uint nworkers = heap->workers()->active_workers(); 256 task_queues()->reserve(nworkers); 257 258 StrongRootsScope scope(nworkers); 259 TaskTerminator terminator(nworkers, task_queues()); 260 ShenandoahFinalMarkingTask<NON_GEN> task(this, &terminator, ShenandoahStringDedup::is_enabled()); 261 heap->workers()->run_task(&task); 262 263 assert(task_queues()->is_empty(), "Should be empty"); 264 } 265 266 267 void ShenandoahConcurrentMark::cancel() { 268 clear(); 269 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); 270 rp->abandon_partial_discovery(); 271 }