1 /*
2 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/barrierSetNMethod.hpp"
28 #include "gc/shared/collectorCounters.hpp"
29 #include "gc/shared/continuationGCSupport.inline.hpp"
30 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
31 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
32 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
33 #include "gc/shenandoah/shenandoahFreeSet.hpp"
34 #include "gc/shenandoah/shenandoahLock.hpp"
35 #include "gc/shenandoah/shenandoahMark.inline.hpp"
36 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
39 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
40 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
41 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
42 #include "gc/shenandoah/shenandoahUtils.hpp"
43 #include "gc/shenandoah/shenandoahVerifier.hpp"
44 #include "gc/shenandoah/shenandoahVMOperations.hpp"
45 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
47 #include "memory/allocation.hpp"
48 #include "prims/jvmtiTagMap.hpp"
49 #include "runtime/vmThread.hpp"
50 #include "utilities/events.hpp"
51
52 // Breakpoint support
53 class ShenandoahBreakpointGCScope : public StackObj {
54 private:
55 const GCCause::Cause _cause;
56 public:
57 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
58 if (cause == GCCause::_wb_breakpoint) {
59 ShenandoahBreakpoint::start_gc();
60 ShenandoahBreakpoint::at_before_gc();
61 }
62 }
63
64 ~ShenandoahBreakpointGCScope() {
65 if (_cause == GCCause::_wb_breakpoint) {
66 ShenandoahBreakpoint::at_after_gc();
67 }
68 }
69 };
70
71 class ShenandoahBreakpointMarkScope : public StackObj {
72 private:
73 const GCCause::Cause _cause;
74 public:
75 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
76 if (_cause == GCCause::_wb_breakpoint) {
77 ShenandoahBreakpoint::at_after_marking_started();
78 }
79 }
80
81 ~ShenandoahBreakpointMarkScope() {
82 if (_cause == GCCause::_wb_breakpoint) {
83 ShenandoahBreakpoint::at_before_marking_completed();
84 }
85 }
86 };
87
88 ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
89 _mark(),
90 _degen_point(ShenandoahDegenPoint::_degenerated_unset) {
91 }
92
93 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
94 return _degen_point;
95 }
96
97 void ShenandoahConcurrentGC::cancel() {
98 ShenandoahConcurrentMark::cancel();
99 }
100
101 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
102 ShenandoahHeap* const heap = ShenandoahHeap::heap();
103 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
104
105 // Reset for upcoming marking
106 entry_reset();
107
108 // Start initial mark under STW
109 vmop_entry_init_mark();
110
111 {
112 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
113 // Concurrent mark roots
114 entry_mark_roots();
115 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false;
116
117 // Continue concurrent mark
118 entry_mark();
119 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false;
120 }
121
122 // Complete marking under STW, and start evacuation
123 vmop_entry_final_mark();
124
125 // Concurrent stack processing
126 if (heap->is_evacuation_in_progress()) {
127 entry_thread_roots();
128 }
129
130 // Process weak roots that might still point to regions that would be broken by cleanup
131 if (heap->is_concurrent_weak_root_in_progress()) {
132 entry_weak_refs();
133 entry_weak_roots();
134 }
135
136 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
137 // the space. This would be the last action if there is nothing to evacuate.
138 entry_cleanup_early();
139
140 {
141 ShenandoahHeapLocker locker(heap->lock());
142 heap->free_set()->log_status();
143 }
144
145 // Perform concurrent class unloading
146 if (heap->unload_classes() &&
147 heap->is_concurrent_weak_root_in_progress()) {
148 entry_class_unloading();
149 }
150
151 // Processing strong roots
152 // This may be skipped if there is nothing to update/evacuate.
153 // If so, strong_root_in_progress would be unset.
154 if (heap->is_concurrent_strong_root_in_progress()) {
155 entry_strong_roots();
156 }
157
158 // Continue the cycle with evacuation and optional update-refs.
159 // This may be skipped if there is nothing to evacuate.
160 // If so, evac_in_progress would be unset by collection set preparation code.
161 if (heap->is_evacuation_in_progress()) {
162 // Concurrently evacuate
163 entry_evacuate();
164 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false;
165
166 // Perform update-refs phase.
167 vmop_entry_init_updaterefs();
168 entry_updaterefs();
169 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
170
171 // Concurrent update thread roots
172 entry_update_thread_roots();
173 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
174
175 vmop_entry_final_updaterefs();
176
177 // Update references freed up collection set, kick the cleanup to reclaim the space.
178 entry_cleanup_complete();
179 } else {
180 vmop_entry_final_roots();
181 }
182
183 return true;
184 }
185
186 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
187 ShenandoahHeap* const heap = ShenandoahHeap::heap();
188 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
189 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
190
191 heap->try_inject_alloc_failure();
192 VM_ShenandoahInitMark op(this);
193 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
194 }
195
196 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
197 ShenandoahHeap* const heap = ShenandoahHeap::heap();
198 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
199 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
200
201 heap->try_inject_alloc_failure();
202 VM_ShenandoahFinalMarkStartEvac op(this);
203 VMThread::execute(&op); // jump to entry_final_mark under safepoint
204 }
205
206 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
207 ShenandoahHeap* const heap = ShenandoahHeap::heap();
208 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
209 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
210
211 heap->try_inject_alloc_failure();
212 VM_ShenandoahInitUpdateRefs op(this);
213 VMThread::execute(&op);
214 }
215
216 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
217 ShenandoahHeap* const heap = ShenandoahHeap::heap();
218 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
219 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
220
221 heap->try_inject_alloc_failure();
222 VM_ShenandoahFinalUpdateRefs op(this);
223 VMThread::execute(&op);
224 }
225
226 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
227 ShenandoahHeap* const heap = ShenandoahHeap::heap();
228 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
229 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
230
231 // This phase does not use workers, no need for setup
232 heap->try_inject_alloc_failure();
233 VM_ShenandoahFinalRoots op(this);
234 VMThread::execute(&op);
235 }
236
237 void ShenandoahConcurrentGC::entry_init_mark() {
238 const char* msg = init_mark_event_message();
239 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
240 EventMark em("%s", msg);
241
242 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
243 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
244 "init marking");
245
246 op_init_mark();
247 }
248
249 void ShenandoahConcurrentGC::entry_final_mark() {
250 const char* msg = final_mark_event_message();
251 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
252 EventMark em("%s", msg);
253
254 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
255 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
256 "final marking");
257
258 op_final_mark();
259 }
260
261 void ShenandoahConcurrentGC::entry_init_updaterefs() {
262 static const char* msg = "Pause Init Update Refs";
263 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
264 EventMark em("%s", msg);
265
266 // No workers used in this phase, no setup required
267 op_init_updaterefs();
268 }
269
270 void ShenandoahConcurrentGC::entry_final_updaterefs() {
271 static const char* msg = "Pause Final Update Refs";
272 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
273 EventMark em("%s", msg);
274
275 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
276 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
277 "final reference update");
278
279 op_final_updaterefs();
280 }
281
282 void ShenandoahConcurrentGC::entry_final_roots() {
283 static const char* msg = "Pause Final Roots";
284 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
285 EventMark em("%s", msg);
286
287 op_final_roots();
288 }
289
290 void ShenandoahConcurrentGC::entry_reset() {
291 ShenandoahHeap* const heap = ShenandoahHeap::heap();
292 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
293 static const char* msg = "Concurrent reset";
294 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
295 EventMark em("%s", msg);
296
297 ShenandoahWorkerScope scope(heap->workers(),
298 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
299 "concurrent reset");
300
301 heap->try_inject_alloc_failure();
302 op_reset();
303 }
304
305 void ShenandoahConcurrentGC::entry_mark_roots() {
306 ShenandoahHeap* const heap = ShenandoahHeap::heap();
307 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
308 const char* msg = "Concurrent marking roots";
309 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
310 EventMark em("%s", msg);
311
312 ShenandoahWorkerScope scope(heap->workers(),
313 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
314 "concurrent marking roots");
315
316 heap->try_inject_alloc_failure();
317 op_mark_roots();
318 }
319
320 void ShenandoahConcurrentGC::entry_mark() {
321 ShenandoahHeap* const heap = ShenandoahHeap::heap();
322 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
323 const char* msg = conc_mark_event_message();
324 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
325 EventMark em("%s", msg);
326
327 ShenandoahWorkerScope scope(heap->workers(),
328 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
329 "concurrent marking");
330
331 heap->try_inject_alloc_failure();
332 op_mark();
333 }
334
335 void ShenandoahConcurrentGC::entry_thread_roots() {
336 ShenandoahHeap* const heap = ShenandoahHeap::heap();
337 static const char* msg = "Concurrent thread roots";
338 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
339 EventMark em("%s", msg);
340
341 ShenandoahWorkerScope scope(heap->workers(),
342 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
343 msg);
344
345 heap->try_inject_alloc_failure();
346 op_thread_roots();
347 }
348
349 void ShenandoahConcurrentGC::entry_weak_refs() {
350 ShenandoahHeap* const heap = ShenandoahHeap::heap();
351 static const char* msg = "Concurrent weak references";
352 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
353 EventMark em("%s", msg);
354
355 ShenandoahWorkerScope scope(heap->workers(),
356 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
357 "concurrent weak references");
358
359 heap->try_inject_alloc_failure();
360 op_weak_refs();
361 }
362
363 void ShenandoahConcurrentGC::entry_weak_roots() {
364 ShenandoahHeap* const heap = ShenandoahHeap::heap();
365 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
366 static const char* msg = "Concurrent weak roots";
367 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
368 EventMark em("%s", msg);
369
370 ShenandoahWorkerScope scope(heap->workers(),
371 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
372 "concurrent weak root");
373
374 heap->try_inject_alloc_failure();
375 op_weak_roots();
376 }
377
378 void ShenandoahConcurrentGC::entry_class_unloading() {
379 ShenandoahHeap* const heap = ShenandoahHeap::heap();
380 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
381 static const char* msg = "Concurrent class unloading";
382 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
383 EventMark em("%s", msg);
384
385 ShenandoahWorkerScope scope(heap->workers(),
386 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
387 "concurrent class unloading");
388
389 heap->try_inject_alloc_failure();
390 op_class_unloading();
391 }
392
393 void ShenandoahConcurrentGC::entry_strong_roots() {
394 ShenandoahHeap* const heap = ShenandoahHeap::heap();
395 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
396 static const char* msg = "Concurrent strong roots";
397 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
398 EventMark em("%s", msg);
399
400 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
401
402 ShenandoahWorkerScope scope(heap->workers(),
403 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
404 "concurrent strong root");
405
406 heap->try_inject_alloc_failure();
407 op_strong_roots();
408 }
409
410 void ShenandoahConcurrentGC::entry_cleanup_early() {
411 ShenandoahHeap* const heap = ShenandoahHeap::heap();
412 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
413 static const char* msg = "Concurrent cleanup";
414 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
415 EventMark em("%s", msg);
416
417 // This phase does not use workers, no need for setup
418 heap->try_inject_alloc_failure();
419 op_cleanup_early();
420 }
421
422 void ShenandoahConcurrentGC::entry_evacuate() {
423 ShenandoahHeap* const heap = ShenandoahHeap::heap();
424 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
425
426 static const char* msg = "Concurrent evacuation";
427 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
428 EventMark em("%s", msg);
429
430 ShenandoahWorkerScope scope(heap->workers(),
431 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
432 "concurrent evacuation");
433
434 heap->try_inject_alloc_failure();
435 op_evacuate();
436 }
437
438 void ShenandoahConcurrentGC::entry_update_thread_roots() {
439 ShenandoahHeap* const heap = ShenandoahHeap::heap();
440 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
441
442 static const char* msg = "Concurrent update thread roots";
443 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
444 EventMark em("%s", msg);
445
446 // No workers used in this phase, no setup required
447 heap->try_inject_alloc_failure();
448 op_update_thread_roots();
449 }
450
451 void ShenandoahConcurrentGC::entry_updaterefs() {
452 ShenandoahHeap* const heap = ShenandoahHeap::heap();
453 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
454 static const char* msg = "Concurrent update references";
455 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
456 EventMark em("%s", msg);
457
458 ShenandoahWorkerScope scope(heap->workers(),
459 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
460 "concurrent reference update");
461
462 heap->try_inject_alloc_failure();
463 op_updaterefs();
464 }
465
466 void ShenandoahConcurrentGC::entry_cleanup_complete() {
467 ShenandoahHeap* const heap = ShenandoahHeap::heap();
468 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
469 static const char* msg = "Concurrent cleanup";
470 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
471 EventMark em("%s", msg);
472
473 // This phase does not use workers, no need for setup
474 heap->try_inject_alloc_failure();
475 op_cleanup_complete();
476 }
477
478 void ShenandoahConcurrentGC::op_reset() {
479 ShenandoahHeap* const heap = ShenandoahHeap::heap();
480 if (ShenandoahPacing) {
481 heap->pacer()->setup_for_reset();
482 }
483
484 heap->prepare_gc();
485 }
486
487 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
488 private:
489 ShenandoahMarkingContext* const _ctx;
490 public:
491 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
492
493 void heap_region_do(ShenandoahHeapRegion* r) {
494 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
495 if (r->is_active()) {
496 // Check if region needs updating its TAMS. We have updated it already during concurrent
497 // reset, so it is very likely we don't need to do another write here.
498 if (_ctx->top_at_mark_start(r) != r->top()) {
499 _ctx->capture_top_at_mark_start(r);
500 }
501 } else {
502 assert(_ctx->top_at_mark_start(r) == r->top(),
503 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
504 }
505 }
506
507 bool is_thread_safe() { return true; }
508 };
509
510 void ShenandoahConcurrentGC::start_mark() {
511 _mark.start_mark();
512 }
513
514 void ShenandoahConcurrentGC::op_init_mark() {
515 ShenandoahHeap* const heap = ShenandoahHeap::heap();
516 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
517 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
518
519 assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap");
520 assert(!heap->marking_context()->is_complete(), "should not be complete");
521 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
522
523 if (ShenandoahVerify) {
524 heap->verifier()->verify_before_concmark();
525 }
526
527 if (VerifyBeforeGC) {
528 Universe::verify();
529 }
530
531 heap->set_concurrent_mark_in_progress(true);
532
533 start_mark();
534
535 {
536 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
537 ShenandoahInitMarkUpdateRegionStateClosure cl;
538 heap->parallel_heap_region_iterate(&cl);
539 }
540
541 // Weak reference processing
542 ShenandoahReferenceProcessor* rp = heap->ref_processor();
543 rp->reset_thread_locals();
544 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
545
546 // Make above changes visible to worker threads
547 OrderAccess::fence();
548 // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
549 // we need to make sure that all its metadata are marked. alternative is to remark
550 // thread roots at final mark pause, but it can be potential latency killer.
551 if (heap->unload_classes()) {
552 ShenandoahCodeRoots::arm_nmethods();
553 }
554
555 ShenandoahStackWatermark::change_epoch_id();
556 if (ShenandoahPacing) {
557 heap->pacer()->setup_for_mark();
558 }
559 }
560
561 void ShenandoahConcurrentGC::op_mark_roots() {
562 _mark.mark_concurrent_roots();
563 }
564
565 void ShenandoahConcurrentGC::op_mark() {
566 _mark.concurrent_mark();
567 }
568
569 void ShenandoahConcurrentGC::op_final_mark() {
570 ShenandoahHeap* const heap = ShenandoahHeap::heap();
571 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
572 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
573
574 if (ShenandoahVerify) {
575 heap->verifier()->verify_roots_no_forwarded();
576 }
577
578 if (!heap->cancelled_gc()) {
579 _mark.finish_mark();
580 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
581
582 // Notify JVMTI that the tagmap table will need cleaning.
583 JvmtiTagMap::set_needs_cleaning();
584
585 heap->prepare_regions_and_collection_set(true /*concurrent*/);
586
587 // Has to be done after cset selection
588 heap->prepare_concurrent_roots();
589
590 if (!heap->collection_set()->is_empty()) {
591 if (ShenandoahVerify) {
592 heap->verifier()->verify_before_evacuation();
593 }
594
595 heap->set_evacuation_in_progress(true);
596 // From here on, we need to update references.
597 heap->set_has_forwarded_objects(true);
598
599 // Verify before arming for concurrent processing.
600 // Otherwise, verification can trigger stack processing.
601 if (ShenandoahVerify) {
602 heap->verifier()->verify_during_evacuation();
603 }
604
605 // Arm nmethods/stack for concurrent processing
606 ShenandoahCodeRoots::arm_nmethods();
607 ShenandoahStackWatermark::change_epoch_id();
608
609 // Notify JVMTI that oops are changed.
610 JvmtiTagMap::set_needs_rehashing();
611
612 if (ShenandoahPacing) {
613 heap->pacer()->setup_for_evac();
614 }
615 } else {
616 if (ShenandoahVerify) {
617 heap->verifier()->verify_after_concmark();
618 }
619
620 if (VerifyAfterGC) {
621 Universe::verify();
622 }
623 }
624 }
625 }
626
627 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
628 private:
629 OopClosure* const _oops;
630
631 public:
632 ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
633 void do_thread(Thread* thread);
634 };
635
636 ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
637 _oops(oops) {
638 }
639
640 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
641 JavaThread* const jt = JavaThread::cast(thread);
642 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
643 }
644
645 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
646 private:
647 ShenandoahJavaThreadsIterator _java_threads;
648
649 public:
650 ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
651 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
652 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
653 }
654
655 void work(uint worker_id) {
656 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
657 // Otherwise, may deadlock with watermark lock
658 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
659 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
660 _java_threads.threads_do(&thr_cl, worker_id);
661 }
662 };
663
664 void ShenandoahConcurrentGC::op_thread_roots() {
665 ShenandoahHeap* const heap = ShenandoahHeap::heap();
666 assert(heap->is_evacuation_in_progress(), "Checked by caller");
667 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
668 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
669 heap->workers()->run_task(&task);
670 }
671
672 void ShenandoahConcurrentGC::op_weak_refs() {
673 ShenandoahHeap* const heap = ShenandoahHeap::heap();
674 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
675 // Concurrent weak refs processing
676 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
677 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
678 ShenandoahBreakpoint::at_after_reference_processing_started();
679 }
680 heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
681 }
682
683 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
684 private:
685 ShenandoahHeap* const _heap;
686 ShenandoahMarkingContext* const _mark_context;
687 bool _evac_in_progress;
688 Thread* const _thread;
689
690 public:
691 ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
692 void do_oop(oop* p);
693 void do_oop(narrowOop* p);
694 };
695
696 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
697 _heap(ShenandoahHeap::heap()),
698 _mark_context(ShenandoahHeap::heap()->marking_context()),
699 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
700 _thread(Thread::current()) {
701 }
702
703 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
704 const oop obj = RawAccess<>::oop_load(p);
705 if (!CompressedOops::is_null(obj)) {
706 if (!_mark_context->is_marked(obj)) {
707 shenandoah_assert_correct(p, obj);
708 ShenandoahHeap::atomic_clear_oop(p, obj);
709 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
710 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
711 if (resolved == obj) {
712 resolved = _heap->evacuate_object(obj, _thread);
713 }
714 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
715 assert(_heap->cancelled_gc() ||
716 _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
717 "Sanity");
718 }
719 }
720 }
721
722 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
723 ShouldNotReachHere();
724 }
725
726 class ShenandoahIsCLDAliveClosure : public CLDClosure {
727 public:
728 void do_cld(ClassLoaderData* cld) {
729 cld->is_alive();
730 }
731 };
732
733 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
734 public:
735 void do_nmethod(nmethod* n) {
736 n->is_unloading();
737 }
738 };
739
740 // This task not only evacuates/updates marked weak roots, but also "NULL"
741 // dead weak roots.
742 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
743 private:
744 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
745
746 // Roots related to concurrent class unloading
747 ShenandoahClassLoaderDataRoots<true /* concurrent */>
748 _cld_roots;
749 ShenandoahConcurrentNMethodIterator _nmethod_itr;
750 ShenandoahPhaseTimings::Phase _phase;
751
752 public:
753 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
754 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
755 _vm_roots(phase),
756 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
757 _nmethod_itr(ShenandoahCodeRoots::table()),
758 _phase(phase) {
759 if (ShenandoahHeap::heap()->unload_classes()) {
760 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
761 _nmethod_itr.nmethods_do_begin();
762 }
763 }
764
765 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
766 if (ShenandoahHeap::heap()->unload_classes()) {
767 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
768 _nmethod_itr.nmethods_do_end();
769 }
770 // Notify runtime data structures of potentially dead oops
771 _vm_roots.report_num_dead();
772 }
773
774 void work(uint worker_id) {
775 ShenandoahConcurrentWorkerSession worker_session(worker_id);
776 {
777 ShenandoahEvacOOMScope oom;
778 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
779 // may race against OopStorage::release() calls.
780 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
781 _vm_roots.oops_do(&cl, worker_id);
782 }
783
784 // If we are going to perform concurrent class unloading later on, we need to
785 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
786 // can cleanup immediate garbage sooner.
787 if (ShenandoahHeap::heap()->unload_classes()) {
788 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
789 // CLD's holder or evacuate it.
790 {
791 ShenandoahIsCLDAliveClosure is_cld_alive;
792 _cld_roots.cld_do(&is_cld_alive, worker_id);
793 }
794
795 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
796 // The closure calls nmethod->is_unloading(). The is_unloading
797 // state is cached, therefore, during concurrent class unloading phase,
798 // we will not touch the metadata of unloading nmethods
799 {
800 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
801 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
802 _nmethod_itr.nmethods_do(&is_nmethod_alive);
803 }
804 }
805 }
806 };
807
808 void ShenandoahConcurrentGC::op_weak_roots() {
809 ShenandoahHeap* const heap = ShenandoahHeap::heap();
810 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
811 // Concurrent weak root processing
812 {
813 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
814 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
815 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
816 heap->workers()->run_task(&task);
817 }
818
819 // Perform handshake to flush out dead oops
820 {
821 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
822 heap->rendezvous_threads();
823 }
824 }
825
826 void ShenandoahConcurrentGC::op_class_unloading() {
827 ShenandoahHeap* const heap = ShenandoahHeap::heap();
828 assert (heap->is_concurrent_weak_root_in_progress() &&
829 heap->unload_classes(),
830 "Checked by caller");
831 heap->do_class_unloading();
832 }
833
834 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
835 private:
836 BarrierSetNMethod* const _bs;
837 ShenandoahEvacuateUpdateMetadataClosure _cl;
838
839 public:
840 ShenandoahEvacUpdateCodeCacheClosure() :
841 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
842 _cl() {
843 }
844
845 void do_nmethod(nmethod* n) {
846 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
847 ShenandoahReentrantLocker locker(data->lock());
848 // Setup EvacOOM scope below reentrant lock to avoid deadlock with
849 // nmethod_entry_barrier
850 ShenandoahEvacOOMScope oom;
851 data->oops_do(&_cl, true/*fix relocation*/);
852 _bs->disarm(n);
853 }
854 };
855
856 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
857 private:
858 ShenandoahPhaseTimings::Phase _phase;
859 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
860 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
861 _cld_roots;
862 ShenandoahConcurrentNMethodIterator _nmethod_itr;
863
864 public:
865 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
866 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
867 _phase(phase),
868 _vm_roots(phase),
869 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
870 _nmethod_itr(ShenandoahCodeRoots::table()) {
871 if (!ShenandoahHeap::heap()->unload_classes()) {
872 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
873 _nmethod_itr.nmethods_do_begin();
874 }
875 }
876
877 ~ShenandoahConcurrentRootsEvacUpdateTask() {
878 if (!ShenandoahHeap::heap()->unload_classes()) {
879 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
880 _nmethod_itr.nmethods_do_end();
881 }
882 }
883
884 void work(uint worker_id) {
885 ShenandoahConcurrentWorkerSession worker_session(worker_id);
886 {
887 ShenandoahEvacOOMScope oom;
888 {
889 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
890 // may race against OopStorage::release() calls.
891 ShenandoahContextEvacuateUpdateRootsClosure cl;
892 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
893 }
894
895 {
896 ShenandoahEvacuateUpdateMetadataClosure cl;
897 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
898 _cld_roots.cld_do(&clds, worker_id);
899 }
900 }
901
902 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
903 if (!ShenandoahHeap::heap()->unload_classes()) {
904 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
905 ShenandoahEvacUpdateCodeCacheClosure cl;
906 _nmethod_itr.nmethods_do(&cl);
907 }
908 }
909 };
910
911 void ShenandoahConcurrentGC::op_strong_roots() {
912 ShenandoahHeap* const heap = ShenandoahHeap::heap();
913 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
914 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
915 heap->workers()->run_task(&task);
916 heap->set_concurrent_strong_root_in_progress(false);
917 }
918
919 void ShenandoahConcurrentGC::op_cleanup_early() {
920 ShenandoahHeap::heap()->free_set()->recycle_trash();
921 }
922
923 void ShenandoahConcurrentGC::op_evacuate() {
924 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
925 }
926
927 void ShenandoahConcurrentGC::op_init_updaterefs() {
928 ShenandoahHeap* const heap = ShenandoahHeap::heap();
929 heap->set_evacuation_in_progress(false);
930 heap->set_concurrent_weak_root_in_progress(false);
931 heap->prepare_update_heap_references(true /*concurrent*/);
932 heap->set_update_refs_in_progress(true);
933
934 if (ShenandoahPacing) {
935 heap->pacer()->setup_for_updaterefs();
936 }
937 }
938
939 void ShenandoahConcurrentGC::op_updaterefs() {
940 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
941 }
942
943 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
944 private:
945 ShenandoahUpdateRefsClosure _cl;
946 public:
947 ShenandoahUpdateThreadClosure();
948 void do_thread(Thread* thread);
949 };
950
951 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
952 HandshakeClosure("Shenandoah Update Thread Roots") {
953 }
954
955 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
956 if (thread->is_Java_thread()) {
957 JavaThread* jt = JavaThread::cast(thread);
958 ResourceMark rm;
959 jt->oops_do(&_cl, NULL);
960 }
961 }
962
963 void ShenandoahConcurrentGC::op_update_thread_roots() {
964 ShenandoahUpdateThreadClosure cl;
965 Handshake::execute(&cl);
966 }
967
968 void ShenandoahConcurrentGC::op_final_updaterefs() {
969 ShenandoahHeap* const heap = ShenandoahHeap::heap();
970 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
971 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
972
973 heap->finish_concurrent_roots();
974
975 // Clear cancelled GC, if set. On cancellation path, the block before would handle
976 // everything.
977 if (heap->cancelled_gc()) {
978 heap->clear_cancelled_gc();
979 }
980
981 // Has to be done before cset is clear
982 if (ShenandoahVerify) {
983 heap->verifier()->verify_roots_in_to_space();
984 }
985
986 heap->update_heap_region_states(true /*concurrent*/);
987
988 heap->set_update_refs_in_progress(false);
989 heap->set_has_forwarded_objects(false);
990
991 if (ShenandoahVerify) {
992 heap->verifier()->verify_after_updaterefs();
993 }
994
995 if (VerifyAfterGC) {
996 Universe::verify();
997 }
998
999 heap->rebuild_free_set(true /*concurrent*/);
1000 }
1001
1002 void ShenandoahConcurrentGC::op_final_roots() {
1003 ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
1004 }
1005
1006 void ShenandoahConcurrentGC::op_cleanup_complete() {
1007 ShenandoahHeap::heap()->free_set()->recycle_trash();
1008 }
1009
1010 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1011 if (ShenandoahHeap::heap()->cancelled_gc()) {
1012 _degen_point = point;
1013 return true;
1014 }
1015 return false;
1016 }
1017
1018 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1019 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1020 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1021 if (heap->unload_classes()) {
1022 return "Pause Init Mark (unload classes)";
1023 } else {
1024 return "Pause Init Mark";
1025 }
1026 }
1027
1028 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1029 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1030 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1031 if (heap->unload_classes()) {
1032 return "Pause Final Mark (unload classes)";
1033 } else {
1034 return "Pause Final Mark";
1035 }
1036 }
1037
1038 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1039 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1040 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1041 if (heap->unload_classes()) {
1042 return "Concurrent marking (unload classes)";
1043 } else {
1044 return "Concurrent marking";
1045 }
1046 }