1 /*
2 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
36 #include "gc/shenandoah/shenandoahGeneration.hpp"
37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
38 #include "gc/shenandoah/shenandoahLock.hpp"
39 #include "gc/shenandoah/shenandoahMark.inline.hpp"
40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
41 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
42 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
45 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
46 #include "gc/shenandoah/shenandoahUtils.hpp"
47 #include "gc/shenandoah/shenandoahVerifier.hpp"
48 #include "gc/shenandoah/shenandoahVMOperations.hpp"
49 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
50 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
51 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
52 #include "memory/allocation.hpp"
53 #include "prims/jvmtiTagMap.hpp"
54 #include "runtime/vmThread.hpp"
55 #include "utilities/events.hpp"
56
57 // Breakpoint support
58 class ShenandoahBreakpointGCScope : public StackObj {
59 private:
60 const GCCause::Cause _cause;
61 public:
62 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
63 if (cause == GCCause::_wb_breakpoint) {
64 ShenandoahBreakpoint::start_gc();
65 ShenandoahBreakpoint::at_before_gc();
66 }
67 }
68
69 ~ShenandoahBreakpointGCScope() {
70 if (_cause == GCCause::_wb_breakpoint) {
71 ShenandoahBreakpoint::at_after_gc();
72 }
73 }
74 };
75
76 class ShenandoahBreakpointMarkScope : public StackObj {
77 private:
78 const GCCause::Cause _cause;
79 public:
80 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
81 if (_cause == GCCause::_wb_breakpoint) {
82 ShenandoahBreakpoint::at_after_marking_started();
83 }
84 }
85
86 ~ShenandoahBreakpointMarkScope() {
87 if (_cause == GCCause::_wb_breakpoint) {
88 ShenandoahBreakpoint::at_before_marking_completed();
89 }
90 }
91 };
92
93 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
94 ShenandoahGC(generation),
95 _mark(generation),
96 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
97 _abbreviated(false),
98 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
99 }
100
101 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
102 return _degen_point;
103 }
104
105 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
106 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
107 const char* msg = conc_init_update_refs_event_message();
108 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
109 EventMark em("%s", msg);
110
111 // Evacuation is complete, retire gc labs and change gc state
112 heap->concurrent_prepare_for_update_refs();
113 }
114
115 void ShenandoahConcurrentGC::entry_update_card_table() {
116 ShenandoahHeap* const heap = ShenandoahHeap::heap();
117 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
118
119 static const char* msg = "Concurrent update cards";
120 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
121 EventMark em("%s", msg);
122
123 ShenandoahWorkerScope scope(heap->workers(),
124 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
125 "concurrent update cards");
126
127 // Heap needs to be parsable here.
128 // Also, parallel heap region iterate must have a phase set.
129 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
130 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
131 }
132
133 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
134 ShenandoahHeap* const heap = ShenandoahHeap::heap();
135 _generation->ref_processor()->set_soft_reference_policy(
136 GCCause::should_clear_all_soft_refs(cause));
137
138 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
139
140 // Reset for upcoming marking
141 entry_reset();
142
143 // Start initial mark under STW
144 vmop_entry_init_mark();
145
146 {
147 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
148
149 // Reset task queue stats here, rather than in mark_concurrent_roots,
150 // because remembered set scan will `push` oops into the queues and
151 // resetting after this happens will lose those counts.
152 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
153
154 // Concurrent remembered set scanning
155 entry_scan_remembered_set();
156
157 // Concurrent mark roots
158 entry_mark_roots();
159 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
160 return false;
161 }
162
163 // Continue concurrent mark
164 entry_mark();
165 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
166 return false;
167 }
168 }
169
170 // Complete marking under STW, and start evacuation
171 vmop_entry_final_mark();
172
173 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
174 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
175 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
176 // from that phase.
177 if (_generation->is_concurrent_mark_in_progress()) {
178 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
179 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
180 return false;
181 }
182
183 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
184
185 // Concurrent stack processing
186 if (heap->is_evacuation_in_progress()) {
187 entry_thread_roots();
188 }
189
190 // Process weak roots that might still point to regions that would be broken by cleanup.
191 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
192 entry_weak_refs();
193 entry_weak_roots();
194
195 // Perform concurrent class unloading before any regions get recycled. Class unloading may
196 // need to inspect unmarked objects in trashed regions.
197 if (heap->unload_classes()) {
198 entry_class_unloading();
199 }
200
201 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
202 // the space. This would be the last action if there is nothing to evacuate. Note that
203 // we will not age young-gen objects in the case that we skip evacuation.
204 entry_cleanup_early();
205
206 heap->free_set()->log_status_under_lock();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Continue the cycle with evacuation and optional update-refs.
216 // This may be skipped if there is nothing to evacuate.
217 // If so, evac_in_progress would be unset by collection set preparation code.
218 if (heap->is_evacuation_in_progress()) {
219 // Roots processing is complete, put the weak roots/ref flags down.
220 vmop_entry_final_roots(false);
221
222 // Concurrently evacuate
223 entry_evacuate();
224 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
225 return false;
226 }
227
228 // Perform update-refs phase.
229 entry_concurrent_update_refs_prepare(heap);
230
231 if (ShenandoahHeap::heap()->mode()->is_generational()) {
232 entry_update_card_table();
233 }
234
235 if (ShenandoahVerify) {
236 vmop_entry_init_update_refs();
237 }
238
239 entry_update_refs();
240 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
241 return false;
242 }
243
244 // Concurrent update thread roots
245 entry_update_thread_roots();
246 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
247 return false;
248 }
249
250 vmop_entry_final_update_refs();
251
252 // Update references freed up collection set, kick the cleanup to reclaim the space.
253 entry_cleanup_complete();
254 } else {
255 _abbreviated = true;
256
257 if (heap->mode()->is_generational()) {
258 entry_complete_abbreviated_cycle();
259
260 // If the promote-in-place operation was cancelled, we can have the degenerated
261 // cycle complete the operation. It will see that no evacuations are in progress,
262 // and that there are regions wanting promotion. The risk with not handling the
263 // cancellation would be failing to restore top for these regions and leaving
264 // them unable to serve allocations for the old generation.
265 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
266 return false;
267 }
268 }
269
270 // In normal cycle, final-update-refs would verify at the end of the cycle.
271 // In abbreviated cycle, we need to verify separately.
272 // This is now also puts the barriers down at the end of the cycle. TODO: Refine.
273 vmop_entry_final_roots(true);
274 }
275
276 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
277 // abbreviated cycle.
278 if (heap->mode()->is_generational()) {
279 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
280 }
281
282 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
283 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
284 // reducing the likelihood that GC will degenerate.
285 entry_reset_after_collect();
286
287 return true;
288 }
289
290 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
291 shenandoah_assert_generational();
292
293 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
294
295 static const char* msg = "Concurrent complete abbreviated cycle";
296 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
297 EventMark em("%s", msg);
298 ShenandoahWorkerScope scope(heap->workers(),
299 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
300 "complete abbreviated");
301
302 // We chose not to evacuate because we found sufficient immediate garbage.
303 // However, there may still be regions to promote in place, so do that now.
304 if (heap->old_generation()->has_in_place_promotions()) {
305 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
306 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
307 heap->promote_regions_in_place(_generation, true);
308 }
309
310 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
311 // the control thread will detect it on its next iteration and run a degenerated young cycle.
312 if (!heap->cancelled_gc() && !_generation->is_old()) {
313 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
314 heap->update_region_ages(_generation->complete_marking_context());
315 }
316 }
317
318 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
319 ShenandoahHeap* const heap = ShenandoahHeap::heap();
320 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
321 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
322
323 heap->try_inject_alloc_failure();
324 VM_ShenandoahInitMark op(this);
325 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
326 }
327
328 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
329 ShenandoahHeap* const heap = ShenandoahHeap::heap();
330 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
331 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
332
333 heap->try_inject_alloc_failure();
334 VM_ShenandoahFinalMarkStartEvac op(this);
335 VMThread::execute(&op); // jump to entry_final_mark under safepoint
336 }
337
338 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
339 ShenandoahHeap* const heap = ShenandoahHeap::heap();
340 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
341 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
342
343 heap->try_inject_alloc_failure();
344 VM_ShenandoahInitUpdateRefs op(this);
345 VMThread::execute(&op);
346 }
347
348 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
349 ShenandoahHeap* const heap = ShenandoahHeap::heap();
350 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
351 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
352
353 heap->try_inject_alloc_failure();
354 VM_ShenandoahFinalUpdateRefs op(this);
355 VMThread::execute(&op);
356 }
357
358 void ShenandoahConcurrentGC::vmop_entry_final_roots(bool at_gc_end) {
359 ShenandoahHeap* const heap = ShenandoahHeap::heap();
360 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
361 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
362
363 // This phase does not use workers, no need for setup
364 heap->try_inject_alloc_failure();
365 VM_ShenandoahFinalRoots op(this, at_gc_end);
366 VMThread::execute(&op);
367 }
368
369 void ShenandoahConcurrentGC::entry_init_mark() {
370 const char* msg = init_mark_event_message();
371 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
372 EventMark em("%s", msg);
373
374 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
375 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
376 "init marking");
377
378 op_init_mark();
379 }
380
381 void ShenandoahConcurrentGC::entry_final_mark() {
382 const char* msg = final_mark_event_message();
383 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
384 EventMark em("%s", msg);
385
386 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
387 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
388 "final marking");
389
390 op_final_mark();
391 }
392
393 void ShenandoahConcurrentGC::entry_init_update_refs() {
394 static const char* msg = "Pause Init Update Refs";
395 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
396 EventMark em("%s", msg);
397
398 // No workers used in this phase, no setup required
399 op_init_update_refs();
400 }
401
402 void ShenandoahConcurrentGC::entry_final_update_refs() {
403 static const char* msg = "Pause Final Update Refs";
404 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
405 EventMark em("%s", msg);
406
407 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
408 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
409 "final reference update");
410
411 op_final_update_refs();
412 }
413
414 void ShenandoahConcurrentGC::entry_reset() {
415 ShenandoahHeap* const heap = ShenandoahHeap::heap();
416 heap->try_inject_alloc_failure();
417
418 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
419 {
420 const char* msg = conc_reset_event_message();
421 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
422 EventMark em("%s", msg);
423
424 ShenandoahWorkerScope scope(heap->workers(),
425 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
426 msg);
427 op_reset();
428 }
429 }
430
431 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
432 if (_generation->is_young()) {
433 ShenandoahHeap* const heap = ShenandoahHeap::heap();
434 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
435 const char* msg = "Concurrent remembered set scanning";
436 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
437 EventMark em("%s", msg);
438
439 ShenandoahWorkerScope scope(heap->workers(),
440 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
441 msg);
442
443 heap->try_inject_alloc_failure();
444 _generation->scan_remembered_set(true /* is_concurrent */);
445 }
446 }
447
448 void ShenandoahConcurrentGC::entry_mark_roots() {
449 ShenandoahHeap* const heap = ShenandoahHeap::heap();
450 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
451 const char* msg = "Concurrent marking roots";
452 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
453 EventMark em("%s", msg);
454
455 ShenandoahWorkerScope scope(heap->workers(),
456 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
457 "concurrent marking roots");
458
459 heap->try_inject_alloc_failure();
460 op_mark_roots();
461 }
462
463 void ShenandoahConcurrentGC::entry_mark() {
464 ShenandoahHeap* const heap = ShenandoahHeap::heap();
465 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
466 const char* msg = conc_mark_event_message();
467 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
468 EventMark em("%s", msg);
469
470 ShenandoahWorkerScope scope(heap->workers(),
471 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
472 "concurrent marking");
473
474 heap->try_inject_alloc_failure();
475 op_mark();
476 }
477
478 void ShenandoahConcurrentGC::entry_thread_roots() {
479 ShenandoahHeap* const heap = ShenandoahHeap::heap();
480 static const char* msg = "Concurrent thread roots";
481 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
482 EventMark em("%s", msg);
483
484 ShenandoahWorkerScope scope(heap->workers(),
485 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
486 msg);
487
488 heap->try_inject_alloc_failure();
489 op_thread_roots();
490 }
491
492 void ShenandoahConcurrentGC::entry_weak_refs() {
493 ShenandoahHeap* const heap = ShenandoahHeap::heap();
494 const char* msg = conc_weak_refs_event_message();
495 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
496 EventMark em("%s", msg);
497
498 ShenandoahWorkerScope scope(heap->workers(),
499 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
500 "concurrent weak references");
501
502 heap->try_inject_alloc_failure();
503 op_weak_refs();
504 }
505
506 void ShenandoahConcurrentGC::entry_weak_roots() {
507 ShenandoahHeap* const heap = ShenandoahHeap::heap();
508 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
509 const char* msg = conc_weak_roots_event_message();
510 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
511 EventMark em("%s", msg);
512
513 ShenandoahWorkerScope scope(heap->workers(),
514 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
515 "concurrent weak root");
516
517 heap->try_inject_alloc_failure();
518 op_weak_roots();
519 }
520
521 void ShenandoahConcurrentGC::entry_class_unloading() {
522 ShenandoahHeap* const heap = ShenandoahHeap::heap();
523 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
524 static const char* msg = "Concurrent class unloading";
525 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
526 EventMark em("%s", msg);
527
528 ShenandoahWorkerScope scope(heap->workers(),
529 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
530 "concurrent class unloading");
531
532 heap->try_inject_alloc_failure();
533 op_class_unloading();
534 }
535
536 void ShenandoahConcurrentGC::entry_strong_roots() {
537 ShenandoahHeap* const heap = ShenandoahHeap::heap();
538 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
539 static const char* msg = "Concurrent strong roots";
540 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
541 EventMark em("%s", msg);
542
543 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
544
545 ShenandoahWorkerScope scope(heap->workers(),
546 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
547 "concurrent strong root");
548
549 heap->try_inject_alloc_failure();
550 op_strong_roots();
551 }
552
553 void ShenandoahConcurrentGC::entry_cleanup_early() {
554 ShenandoahHeap* const heap = ShenandoahHeap::heap();
555 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
556 const char* msg = conc_cleanup_event_message();
557 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
558 EventMark em("%s", msg);
559
560 // This phase does not use workers, no need for setup
561 heap->try_inject_alloc_failure();
562 op_cleanup_early();
563 if (!heap->is_evacuation_in_progress()) {
564 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
565 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
566 // during promote-in-place processing.
567 heap->rebuild_free_set(true /*concurrent*/);
568 }
569 }
570
571 void ShenandoahConcurrentGC::entry_evacuate() {
572 ShenandoahHeap* const heap = ShenandoahHeap::heap();
573 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
574
575 static const char* msg = "Concurrent evacuation";
576 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
577 EventMark em("%s", msg);
578
579 ShenandoahWorkerScope scope(heap->workers(),
580 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
581 "concurrent evacuation");
582
583 heap->try_inject_alloc_failure();
584 op_evacuate();
585 }
586
587 void ShenandoahConcurrentGC::entry_update_thread_roots() {
588 ShenandoahHeap* const heap = ShenandoahHeap::heap();
589 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
590
591 static const char* msg = "Concurrent update thread roots";
592 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
593 EventMark em("%s", msg);
594
595 // No workers used in this phase, no setup required
596 heap->try_inject_alloc_failure();
597 op_update_thread_roots();
598 }
599
600 void ShenandoahConcurrentGC::entry_update_refs() {
601 ShenandoahHeap* const heap = ShenandoahHeap::heap();
602 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
603 static const char* msg = "Concurrent update references";
604 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
605 EventMark em("%s", msg);
606
607 ShenandoahWorkerScope scope(heap->workers(),
608 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
609 "concurrent reference update");
610
611 heap->try_inject_alloc_failure();
612 op_update_refs();
613 }
614
615 void ShenandoahConcurrentGC::entry_cleanup_complete() {
616 ShenandoahHeap* const heap = ShenandoahHeap::heap();
617 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
618 const char* msg = conc_cleanup_event_message();
619 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
620 EventMark em("%s", msg);
621
622 // This phase does not use workers, no need for setup
623 heap->try_inject_alloc_failure();
624 op_cleanup_complete();
625 }
626
627 void ShenandoahConcurrentGC::entry_reset_after_collect() {
628 ShenandoahHeap* const heap = ShenandoahHeap::heap();
629 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
630 const char* msg = conc_reset_after_collect_event_message();
631 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
632 EventMark em("%s", msg);
633
634 op_reset_after_collect();
635 }
636
637 void ShenandoahConcurrentGC::op_reset() {
638 ShenandoahHeap* const heap = ShenandoahHeap::heap();
639
640 // If it is old GC bootstrap cycle, always clear bitmap for global gen
641 // to ensure bitmap for old gen is clear for old GC cycle after this.
642 if (_do_old_gc_bootstrap) {
643 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
644 heap->global_generation()->prepare_gc();
645 } else {
646 _generation->prepare_gc();
647 }
648
649 if (heap->mode()->is_generational()) {
650 heap->old_generation()->card_scan()->mark_read_table_as_clean();
651 }
652 }
653
654 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
655 private:
656 ShenandoahMarkingContext* const _ctx;
657 public:
658 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
659
660 void heap_region_do(ShenandoahHeapRegion* r) {
661 assert(!r->has_live(), "Region %zu should have no live data", r->index());
662 if (r->is_active()) {
663 // Check if region needs updating its TAMS. We have updated it already during concurrent
664 // reset, so it is very likely we don't need to do another write here. Since most regions
665 // are not "active", this path is relatively rare.
666 if (_ctx->top_at_mark_start(r) != r->top()) {
667 _ctx->capture_top_at_mark_start(r);
668 }
669 } else {
670 assert(_ctx->top_at_mark_start(r) == r->top(),
671 "Region %zu should already have correct TAMS", r->index());
672 }
673 }
674
675 bool is_thread_safe() { return true; }
676 };
677
678 void ShenandoahConcurrentGC::start_mark() {
679 _mark.start_mark();
680 }
681
682 void ShenandoahConcurrentGC::op_init_mark() {
683 ShenandoahHeap* const heap = ShenandoahHeap::heap();
684 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
685 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
686
687 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
688 assert(!_generation->is_mark_complete(), "should not be complete");
689 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
690
691 // First pause in cycle, check that barriers were not left enabled.
692 ShenandoahCodeRoots::check_barriers();
693
694 if (heap->mode()->is_generational()) {
695 if (_generation->is_global()) {
696 heap->old_generation()->cancel_gc();
697 }
698
699 {
700 // After we swap card table below, the write-table is all clean, and the read table holds
701 // cards dirty prior to the start of GC. Young and bootstrap collection will update
702 // the write card table as a side effect of remembered set scanning. Global collection will
703 // update the card table as a side effect of global marking of old objects.
704 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
705 _generation->swap_card_tables();
706 }
707 }
708
709 if (ShenandoahVerify) {
710 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
711 heap->verifier()->verify_before_concmark(_generation);
712 }
713
714 if (VerifyBeforeGC) {
715 Universe::verify();
716 }
717
718 _generation->set_concurrent_mark_in_progress(true);
719
720 start_mark();
721
722 if (_do_old_gc_bootstrap) {
723 shenandoah_assert_generational();
724 // Update region state for both young and old regions
725 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
726 ShenandoahInitMarkUpdateRegionStateClosure cl;
727 heap->parallel_heap_region_iterate(&cl);
728 heap->old_generation()->ref_processor()->reset_thread_locals();
729 } else {
730 // Update region state for only young regions
731 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
732 ShenandoahInitMarkUpdateRegionStateClosure cl;
733 _generation->parallel_heap_region_iterate(&cl);
734 }
735
736 // Weak reference processing
737 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
738 rp->reset_thread_locals();
739
740 // Make above changes visible to worker threads
741 OrderAccess::fence();
742
743 // Arm nmethods/stack for concurrent processing
744 ShenandoahCodeRoots::arm_nmethods();
745 ShenandoahStackWatermark::change_epoch_id();
746
747 {
748 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
749 heap->propagate_gc_state_to_all_threads();
750 }
751 }
752
753 void ShenandoahConcurrentGC::op_mark_roots() {
754 _mark.mark_concurrent_roots();
755 }
756
757 void ShenandoahConcurrentGC::op_mark() {
758 _mark.concurrent_mark();
759 }
760
761 void ShenandoahConcurrentGC::op_final_mark() {
762 ShenandoahHeap* const heap = ShenandoahHeap::heap();
763 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
764 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
765
766 if (ShenandoahVerify) {
767 heap->verifier()->verify_roots_no_forwarded(_generation);
768 }
769
770 if (!heap->cancelled_gc()) {
771 _mark.finish_mark();
772 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
773
774 // Notify JVMTI that the tagmap table will need cleaning.
775 JvmtiTagMap::set_needs_cleaning();
776
777 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
778 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
779 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
780 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
781
782 // Has to be done after cset selection
783 heap->prepare_concurrent_roots();
784
785 if (!heap->collection_set()->is_empty()) {
786 LogTarget(Debug, gc, cset) lt;
787 if (lt.is_enabled()) {
788 ResourceMark rm;
789 LogStream ls(lt);
790 heap->collection_set()->print_on(&ls);
791 }
792
793 if (ShenandoahVerify) {
794 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
795 heap->verifier()->verify_before_evacuation(_generation);
796 }
797
798 heap->set_evacuation_in_progress(true);
799 // From here on, we need to update references.
800 heap->set_has_forwarded_objects(true);
801 } else {
802 if (ShenandoahVerify) {
803 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
804 if (has_in_place_promotions(heap)) {
805 heap->verifier()->verify_after_concmark_with_promotions(_generation);
806 } else {
807 heap->verifier()->verify_after_concmark(_generation);
808 }
809 }
810 }
811 }
812
813 // Arm nmethods/stack for concurrent processing
814 ShenandoahCodeRoots::arm_nmethods();
815 ShenandoahStackWatermark::change_epoch_id();
816
817 {
818 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
819 heap->propagate_gc_state_to_all_threads();
820 }
821 }
822
823 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
824 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
825 }
826
827 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
828 private:
829 OopClosure* const _oops;
830 public:
831 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
832
833 void do_thread(Thread* thread) override {
834 JavaThread* const jt = JavaThread::cast(thread);
835 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
836 }
837 };
838
839 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
840 private:
841 ShenandoahJavaThreadsIterator _java_threads;
842
843 public:
844 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
845 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
846 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
847 }
848
849 void work(uint worker_id) override {
850 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
851 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
852 _java_threads.threads_do(&thr_cl, worker_id);
853 }
854 };
855
856 void ShenandoahConcurrentGC::op_thread_roots() {
857 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
858 assert(heap->is_evacuation_in_progress(), "Checked by caller");
859 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
860 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
861 heap->workers()->run_task(&task);
862 }
863
864 void ShenandoahConcurrentGC::op_weak_refs() {
865 ShenandoahHeap* const heap = ShenandoahHeap::heap();
866 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
867 // Concurrent weak refs processing
868 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
869 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
870 ShenandoahBreakpoint::at_after_reference_processing_started();
871 }
872 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
873 }
874
875 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
876 private:
877 ShenandoahHeap* const _heap;
878 ShenandoahGeneration* const _generation;
879 ShenandoahMarkingContext* const _mark_context;
880 bool _evac_in_progress;
881 Thread* const _thread;
882
883 public:
884 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
885 void do_oop(oop* p);
886 void do_oop(narrowOop* p);
887 };
888
889 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
890 _heap(ShenandoahHeap::heap()),
891 _generation(generation),
892 _mark_context(ShenandoahHeap::heap()->marking_context()),
893 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
894 _thread(Thread::current()) {
895 }
896
897 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
898 const oop obj = RawAccess<>::oop_load(p);
899 if (!CompressedOops::is_null(obj)) {
900 if (!_mark_context->is_marked(obj)) {
901 if (_generation->contains(obj)) {
902 // Note: The obj is dead here. Do not touch it, just clear.
903 ShenandoahHeap::atomic_clear_oop(p, obj);
904 }
905 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
906 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
907 if (resolved == obj) {
908 resolved = _heap->evacuate_object(obj, _thread);
909 }
910 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
911 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
912 }
913 }
914 }
915
916 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
917 ShouldNotReachHere();
918 }
919
920 class ShenandoahIsCLDAliveClosure : public CLDClosure {
921 public:
922 void do_cld(ClassLoaderData* cld) {
923 cld->is_alive();
924 }
925 };
926
927 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
928 public:
929 void do_nmethod(nmethod* n) {
930 n->is_unloading();
931 }
932 };
933
934 // This task not only evacuates/updates marked weak roots, but also "null"
935 // dead weak roots.
936 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
937 private:
938 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
939
940 // Roots related to concurrent class unloading
941 ShenandoahClassLoaderDataRoots<true /* concurrent */>
942 _cld_roots;
943 ShenandoahConcurrentNMethodIterator _nmethod_itr;
944 ShenandoahGeneration* _generation;
945 ShenandoahPhaseTimings::Phase _phase;
946
947 public:
948 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
949 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
950 _vm_roots(phase),
951 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
952 _nmethod_itr(ShenandoahCodeRoots::table()),
953 _generation(generation),
954 _phase(phase) {}
955
956 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
957 // Notify runtime data structures of potentially dead oops
958 _vm_roots.report_num_dead();
959 }
960
961 void work(uint worker_id) override {
962 ShenandoahConcurrentWorkerSession worker_session(worker_id);
963 SuspendibleThreadSetJoiner sts_join;
964 {
965 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
966 // may race against OopStorage::release() calls.
967 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
968 _vm_roots.oops_do(&cl, worker_id);
969 }
970
971 // If we are going to perform concurrent class unloading later on, we need to
972 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
973 // can clean up immediate garbage sooner.
974 if (ShenandoahHeap::heap()->unload_classes()) {
975 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
976 // CLD's holder or evacuate it.
977 {
978 ShenandoahIsCLDAliveClosure is_cld_alive;
979 _cld_roots.cld_do(&is_cld_alive, worker_id);
980 }
981
982 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
983 // The closure calls nmethod->is_unloading(). The is_unloading
984 // state is cached, therefore, during concurrent class unloading phase,
985 // we will not touch the metadata of unloading nmethods
986 {
987 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
988 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
989 _nmethod_itr.nmethods_do(&is_nmethod_alive);
990 }
991 }
992 }
993 };
994
995 void ShenandoahConcurrentGC::op_weak_roots() {
996 ShenandoahHeap* const heap = ShenandoahHeap::heap();
997 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
998 {
999 // Concurrent weak root processing
1000 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1001 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1002 heap->workers()->run_task(&task);
1003 }
1004
1005 {
1006 // It is possible for mutators executing the load reference barrier to have
1007 // loaded an oop through a weak handle that has since been nulled out by
1008 // weak root processing. Handshaking here forces them to complete the
1009 // barrier before the GC cycle continues and does something that would
1010 // change the evaluation of the barrier (for example, resetting the TAMS
1011 // on trashed regions could make an oop appear to be marked _after_ the
1012 // region has been recycled).
1013 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1014 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1015 }
1016 }
1017
1018 void ShenandoahConcurrentGC::op_class_unloading() {
1019 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1020 assert (heap->is_concurrent_weak_root_in_progress() &&
1021 heap->unload_classes(),
1022 "Checked by caller");
1023 heap->do_class_unloading();
1024 }
1025
1026 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1027 private:
1028 ShenandoahEvacuateUpdateMetadataClosure _cl;
1029
1030 public:
1031 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1032
1033 void do_nmethod(nmethod* n) {
1034 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1035 ShenandoahNMethodLocker locker(data->lock());
1036 data->oops_do(&_cl, /* fix_relocations = */ true);
1037 ShenandoahNMethod::complete_and_disarm_nmethod_unlocked(n);
1038 }
1039 };
1040
1041 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1042 private:
1043 ShenandoahPhaseTimings::Phase _phase;
1044 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1045 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1046 _cld_roots;
1047 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1048
1049 public:
1050 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1051 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1052 _phase(phase),
1053 _vm_roots(phase),
1054 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1055 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1056
1057 void work(uint worker_id) {
1058 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1059 {
1060 {
1061 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1062 // may race against OopStorage::release() calls.
1063 ShenandoahContextEvacuateUpdateRootsClosure cl;
1064 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1065 }
1066
1067 {
1068 ShenandoahEvacuateUpdateMetadataClosure cl;
1069 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1070 _cld_roots.cld_do(&clds, worker_id);
1071 }
1072 }
1073
1074 if (!ShenandoahHeap::heap()->unload_classes()) {
1075 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1076 ShenandoahEvacUpdateCodeCacheClosure cl;
1077 _nmethod_itr.nmethods_do(&cl);
1078 }
1079 }
1080 };
1081
1082 void ShenandoahConcurrentGC::op_strong_roots() {
1083 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1084 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1085 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1086 heap->workers()->run_task(&task);
1087 heap->set_concurrent_strong_root_in_progress(false);
1088 }
1089
1090 void ShenandoahConcurrentGC::op_cleanup_early() {
1091 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1092 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1093 "cleanup early.");
1094 ShenandoahHeap::heap()->recycle_trash();
1095 }
1096
1097 void ShenandoahConcurrentGC::op_evacuate() {
1098 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1099 }
1100
1101 void ShenandoahConcurrentGC::op_init_update_refs() {
1102 if (ShenandoahVerify) {
1103 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1104 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1105 heap->verifier()->verify_before_update_refs(_generation);
1106 }
1107 }
1108
1109 void ShenandoahConcurrentGC::op_update_refs() {
1110 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1111 }
1112
1113 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1114 private:
1115 // This closure runs when thread is stopped for handshake, which means
1116 // we can use non-concurrent closure here, as long as it only updates
1117 // locations modified by the thread itself, i.e. stack locations.
1118 ShenandoahNonConcUpdateRefsClosure _cl;
1119 public:
1120 ShenandoahUpdateThreadHandshakeClosure();
1121 void do_thread(Thread* thread) override;
1122 };
1123
1124 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1125 HandshakeClosure("Shenandoah Update Thread Roots") {
1126 }
1127
1128 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1129 if (thread->is_Java_thread()) {
1130 JavaThread* jt = JavaThread::cast(thread);
1131 ResourceMark rm;
1132 jt->oops_do(&_cl, nullptr);
1133 }
1134 }
1135
1136 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1137 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1138 // to pick up overwritten pointers that point into a cset region. If these pointers
1139 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1140 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1141 //
1142 // Therefore, at the end of update refs, we use this closure to update the thread roots
1143 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1144 // anything that has already been marked or anything that points to a region which is
1145 // not old. We do not need to worry about ABA situations where a region may become old
1146 // after the pointer is enqueued but before it is filtered. There are only two ways a
1147 // region may become old:
1148 // 1. The region is promoted in place. This is safe because such regions will never
1149 // be in the collection set. If this happens, the pointer will be preserved, essentially
1150 // becoming part of the old snapshot.
1151 // 2. The region is allocated during evacuation of old. This is also not a concern because
1152 // we haven't yet finished marking old so no mixed evacuations will happen.
1153 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1154 ShenandoahFlushSATB _flush_all_satb;
1155
1156 public:
1157 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1158 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1159 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1160 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1161 "Should be filtering pointers outside of old during old marking");
1162 }
1163
1164 void do_thread(Thread* thread) override {
1165 _update_roots.do_thread(thread);
1166 _flush_all_satb.do_thread(thread);
1167 }
1168 };
1169
1170 void ShenandoahConcurrentGC::op_update_thread_roots() {
1171 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1172 if (heap->is_concurrent_old_mark_in_progress()) {
1173 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1174 Handshake::execute(&cl);
1175 } else {
1176 ShenandoahUpdateThreadHandshakeClosure cl;
1177 Handshake::execute(&cl);
1178 }
1179 }
1180
1181 void ShenandoahConcurrentGC::op_final_update_refs() {
1182 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1183 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1184 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1185
1186 heap->finish_concurrent_roots();
1187
1188 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1189 // everything.
1190 if (heap->cancelled_gc()) {
1191 heap->clear_cancelled_gc();
1192 }
1193
1194 // Has to be done before cset is clear
1195 if (ShenandoahVerify) {
1196 heap->verifier()->verify_roots_in_to_space(_generation);
1197 }
1198
1199 // If we are running in generational mode, this will also age active regions that
1200 // haven't been used for allocation.
1201 heap->update_heap_region_states(true /*concurrent*/);
1202
1203 heap->set_update_refs_in_progress(false);
1204 heap->set_has_forwarded_objects(false);
1205
1206 if (ShenandoahVerify) {
1207 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1208 heap->verifier()->verify_after_update_refs(_generation);
1209 }
1210
1211 if (VerifyAfterGC) {
1212 Universe::verify();
1213 }
1214
1215 heap->rebuild_free_set(true /*concurrent*/);
1216 _generation->heuristics()->start_idle_span();
1217
1218 {
1219 // Final pause: update GC barriers to idle state.
1220 ShenandoahCodeRoots::arm_nmethods();
1221 ShenandoahStackWatermark::change_epoch_id();
1222 }
1223
1224 {
1225 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1226 heap->propagate_gc_state_to_all_threads();
1227 }
1228 }
1229
1230 void ShenandoahConcurrentGC::entry_final_roots(bool at_gc_end) {
1231 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1232 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1233
1234 const char* msg = final_roots_event_message();
1235 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1236 EventMark em("%s", msg);
1237 ShenandoahWorkerScope scope(heap->workers(),
1238 ParallelGCThreads,
1239 msg);
1240
1241 heap->op_final_roots(at_gc_end);
1242 }
1243
1244 void ShenandoahConcurrentGC::op_cleanup_complete() {
1245 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1246 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1247 "cleanup complete.");
1248 ShenandoahHeap::heap()->recycle_trash();
1249 }
1250
1251 void ShenandoahConcurrentGC::op_reset_after_collect() {
1252 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1253 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1254 "reset after collection.");
1255
1256 // Final concurrent phase: complete disabling all barriers.
1257 ShenandoahCodeRoots::disarm_nmethods();
1258
1259 // Check that barriers were not left enabled.
1260 ShenandoahCodeRoots::check_barriers();
1261
1262 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1263 if (heap->mode()->is_generational()) {
1264 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1265 // the young generation intact. In particular, reference processing in the old generation may potentially
1266 // need the reachability of a young generation referent of a Reference object in the old generation.
1267 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1268 heap->young_generation()->reset_mark_bitmap<false>();
1269 }
1270 } else {
1271 _generation->reset_mark_bitmap<false>();
1272 }
1273 }
1274
1275 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1276 if (ShenandoahHeap::heap()->cancelled_gc()) {
1277 _degen_point = point;
1278 return true;
1279 }
1280 return false;
1281 }
1282
1283 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1284 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1285 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1286 if (heap->unload_classes()) {
1287 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1288 } else {
1289 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1290 }
1291 }
1292
1293 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1294 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1295 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1296 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1297
1298 if (heap->unload_classes()) {
1299 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1300 } else {
1301 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1302 }
1303 }
1304
1305 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1306 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1307 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1308 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1309 if (heap->unload_classes()) {
1310 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1311 } else {
1312 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1313 }
1314 }
1315
1316 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1317 if (ShenandoahHeap::heap()->unload_classes()) {
1318 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1319 } else {
1320 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1321 }
1322 }
1323
1324 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1325 if (ShenandoahHeap::heap()->unload_classes()) {
1326 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1327 } else {
1328 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1329 }
1330 }
1331
1332 const char* ShenandoahConcurrentGC::final_roots_event_message() const {
1333 if (ShenandoahHeap::heap()->unload_classes()) {
1334 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
1335 } else {
1336 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
1337 }
1338 }
1339
1340 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1341 if (ShenandoahHeap::heap()->unload_classes()) {
1342 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1343 } else {
1344 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1345 }
1346 }
1347
1348 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1349 if (ShenandoahHeap::heap()->unload_classes()) {
1350 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1351 } else {
1352 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1353 }
1354 }
1355
1356 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1357 if (ShenandoahHeap::heap()->unload_classes()) {
1358 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1359 } else {
1360 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1361 }
1362 }
1363
1364 const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
1365 if (ShenandoahHeap::heap()->unload_classes()) {
1366 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
1367 } else {
1368 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
1369 }
1370 }