1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
37 #include "gc/shenandoah/shenandoahGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahLock.hpp"
40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
42 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
52 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
53 #include "memory/allocation.hpp"
54 #include "prims/jvmtiTagMap.hpp"
55 #include "runtime/vmThread.hpp"
56 #include "utilities/events.hpp"
57
58 // Breakpoint support
59 class ShenandoahBreakpointGCScope : public StackObj {
60 private:
61 const GCCause::Cause _cause;
62 public:
63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
64 if (cause == GCCause::_wb_breakpoint) {
65 ShenandoahBreakpoint::start_gc();
66 ShenandoahBreakpoint::at_before_gc();
67 }
68 }
69
70 ~ShenandoahBreakpointGCScope() {
71 if (_cause == GCCause::_wb_breakpoint) {
72 ShenandoahBreakpoint::at_after_gc();
73 }
74 }
75 };
76
77 class ShenandoahBreakpointMarkScope : public StackObj {
78 private:
79 const GCCause::Cause _cause;
80 public:
81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
82 if (_cause == GCCause::_wb_breakpoint) {
83 ShenandoahBreakpoint::at_after_marking_started();
84 }
85 }
86
87 ~ShenandoahBreakpointMarkScope() {
88 if (_cause == GCCause::_wb_breakpoint) {
89 ShenandoahBreakpoint::at_before_marking_completed();
90 }
91 }
92 };
93
94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
95 ShenandoahGC(generation),
96 _mark(generation),
97 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
98 _abbreviated(false),
99 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
100 }
101
102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
103 return _degen_point;
104 }
105
106 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
107 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
108 const char* msg = conc_init_update_refs_event_message();
109 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
110 EventMark em("%s", msg);
111
112 heap->try_inject_pin();
113 // Evacuation is complete, retire gc labs and change gc state
114 heap->concurrent_prepare_for_update_refs();
115 }
116
117 void ShenandoahConcurrentGC::entry_update_card_table() {
118 ShenandoahHeap* const heap = ShenandoahHeap::heap();
119 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
120
121 static const char* msg = "Concurrent update cards";
122 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
123 EventMark em("%s", msg);
124
125 ShenandoahWorkerScope scope(heap->workers(),
126 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
127 "concurrent update cards");
128
129 heap->try_inject_pin();
130 // Heap needs to be parsable here.
131 // Also, parallel heap region iterate must have a phase set.
132 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
133 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
134 }
135
136 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
137 ShenandoahHeap* const heap = ShenandoahHeap::heap();
138 _generation->ref_processor()->set_soft_reference_policy(
139 GCCause::should_clear_all_soft_refs(cause));
140
141 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
142
143 // Reset for upcoming marking
144 entry_reset();
145
146 // Start initial mark under STW
147 vmop_entry_init_mark();
148
149 {
150 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
151
152 // Reset task queue stats here, rather than in mark_concurrent_roots,
153 // because remembered set scan will `push` oops into the queues and
154 // resetting after this happens will lose those counts.
155 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
156
157 // Concurrent remembered set scanning
158 entry_scan_remembered_set();
159
160 // Concurrent mark roots
161 entry_mark_roots();
162 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
163 return false;
164 }
165
166 // Continue concurrent mark
167 entry_mark();
168 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
169 return false;
170 }
171 }
172
173 // Complete marking under STW, and start evacuation
174 vmop_entry_final_mark();
175
176 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
177 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
178 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
179 // from that phase.
180 if (_generation->is_concurrent_mark_in_progress()) {
181 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
182 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
183 return false;
184 }
185
186 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
187
188 // Finish all thread/stack roots if needed. This completes stack watermark processing.
189 if (heap->is_evacuation_in_progress()) {
190 entry_thread_roots();
191 }
192
193 // Process weak roots that might still point to regions that would be broken by cleanup.
194 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
195 entry_weak_refs();
196 entry_weak_roots();
197
198 // Perform concurrent class unloading before any regions get recycled. Class unloading may
199 // need to inspect unmarked objects in trashed regions.
200 if (heap->unload_classes()) {
201 entry_class_unloading();
202 }
203
204 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
205 // the space. This would be the last action if there is nothing to evacuate. Note that
206 // we will not age young-gen objects in the case that we skip evacuation.
207 entry_cleanup_early();
208
209 // Processing strong roots
210 // This may be skipped if there is nothing to update/evacuate.
211 // If so, strong_root_in_progress would be unset.
212 if (heap->is_concurrent_strong_root_in_progress()) {
213 entry_strong_roots();
214 }
215
216 // Roots processing is complete, put the weak roots flag down.
217 entry_final_roots();
218
219 // Continue the cycle with evacuation and optional update-refs.
220 // This may be skipped if there is nothing to evacuate.
221 // If so, evac_in_progress would be unset by collection set preparation code.
222 if (heap->is_evacuation_in_progress()) {
223 // Concurrently evacuate
224 entry_evacuate();
225 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
226 return false;
227 }
228
229 // Perform update-refs phase.
230 entry_concurrent_update_refs_prepare(heap);
231
232 if (ShenandoahHeap::heap()->mode()->is_generational()) {
233 entry_update_card_table();
234 }
235
236 if (ShenandoahVerify) {
237 vmop_entry_init_update_refs();
238 }
239
240 entry_update_refs();
241 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
242 return false;
243 }
244
245 // Concurrent update thread roots
246 entry_update_thread_roots();
247 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
248 return false;
249 }
250
251 vmop_entry_final_update_refs();
252
253 // Update references freed up collection set, kick the cleanup to reclaim the space.
254 entry_cleanup_complete();
255 } else {
256 _abbreviated = true;
257
258 if (heap->mode()->is_generational()) {
259 entry_complete_abbreviated_cycle();
260
261 // If the promote-in-place operation was cancelled, we can have the degenerated
262 // cycle complete the operation. It will see that no evacuations are in progress,
263 // and that there are regions wanting promotion. The risk with not handling the
264 // cancellation would be failing to restore top for these regions and leaving
265 // them unable to serve allocations for the old generation.
266 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
267 return false;
268 }
269 }
270
271 // In normal cycle, final-update-refs would verify at the end of the cycle.
272 // In abbreviated cycle, we need to verify separately.
273 if (ShenandoahVerify) {
274 vmop_entry_final_verify();
275 }
276 }
277
278 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
279 // abbreviated cycle.
280 if (heap->mode()->is_generational()) {
281 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
282 }
283
284 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
285 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
286 // reducing the likelihood that GC will degenerate.
287 entry_reset_after_collect();
288
289 return true;
290 }
291
292 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
293 shenandoah_assert_generational();
294
295 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
296
297 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
298 static const char* msg = "Concurrent complete abbreviated cycle";
299 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
300 EventMark em("%s", msg);
301
302 ShenandoahWorkerScope scope(heap->workers(),
303 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
304 msg);
305
306 heap->try_inject_pin();
307 // We chose not to evacuate because we found sufficient immediate garbage.
308 // However, there may still be regions to promote in place, so do that now.
309 if (heap->old_generation()->has_in_place_promotions()) {
310 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
311 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
312 heap->promote_regions_in_place(_generation, true);
313 }
314
315 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
316 // the control thread will detect it on its next iteration and run a degenerated young cycle.
317 if (!heap->cancelled_gc() && !_generation->is_old()) {
318 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
319 heap->update_region_ages(_generation->complete_marking_context());
320 }
321 }
322
323 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
324 ShenandoahHeap* const heap = ShenandoahHeap::heap();
325 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
326 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
327
328 heap->try_inject_alloc_failure();
329 VM_ShenandoahInitMark op(this);
330 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
331 }
332
333 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
334 ShenandoahHeap* const heap = ShenandoahHeap::heap();
335 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
336 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
337
338 heap->try_inject_alloc_failure();
339 VM_ShenandoahFinalMarkStartEvac op(this);
340 VMThread::execute(&op); // jump to entry_final_mark under safepoint
341 heap->try_inject_pin();
342 }
343
344 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
345 ShenandoahHeap* const heap = ShenandoahHeap::heap();
346 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
347 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
348
349 heap->try_inject_alloc_failure();
350 heap->try_inject_pin();
351 VM_ShenandoahInitUpdateRefs op(this);
352 VMThread::execute(&op);
353 }
354
355 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
356 ShenandoahHeap* const heap = ShenandoahHeap::heap();
357 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
358 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
359
360 heap->try_inject_alloc_failure();
361 heap->try_inject_pin();
362 VM_ShenandoahFinalUpdateRefs op(this);
363 VMThread::execute(&op);
364 }
365
366 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
367 ShenandoahHeap* const heap = ShenandoahHeap::heap();
368 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
369 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
370
371 // This phase does not use workers, no need for setup
372 heap->try_inject_alloc_failure();
373 heap->try_inject_pin();
374 VM_ShenandoahFinalVerify op(this);
375 VMThread::execute(&op);
376 }
377
378 void ShenandoahConcurrentGC::entry_init_mark() {
379 const char* msg = init_mark_event_message();
380 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
381 EventMark em("%s", msg);
382
383 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
384 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
385 "init marking");
386
387 op_init_mark();
388 }
389
390 void ShenandoahConcurrentGC::entry_final_mark() {
391 const char* msg = final_mark_event_message();
392 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
393 EventMark em("%s", msg);
394
395 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
396 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
397 "final marking");
398
399 op_final_mark();
400 }
401
402 void ShenandoahConcurrentGC::entry_init_update_refs() {
403 static const char* msg = "Pause Init Update Refs";
404 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
405 EventMark em("%s", msg);
406
407 // No workers used in this phase, no setup required
408 op_init_update_refs();
409 }
410
411 void ShenandoahConcurrentGC::entry_final_update_refs() {
412 static const char* msg = "Pause Final Update Refs";
413 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
414 EventMark em("%s", msg);
415
416 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
417 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
418 "final reference update");
419
420 op_final_update_refs();
421 }
422
423 void ShenandoahConcurrentGC::entry_final_verify() {
424 const char* msg = verify_final_event_message();
425 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
426 EventMark em("%s", msg);
427
428 op_verify_final();
429 }
430
431 void ShenandoahConcurrentGC::entry_reset() {
432 ShenandoahHeap* const heap = ShenandoahHeap::heap();
433 heap->release_injected_pins();
434 heap->try_inject_alloc_failure();
435
436 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
437 {
438 const char* msg = conc_reset_event_message();
439 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
440 EventMark em("%s", msg);
441
442 ShenandoahWorkerScope scope(heap->workers(),
443 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
444 msg);
445 op_reset();
446 }
447 }
448
449 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
450 if (_generation->is_young()) {
451 ShenandoahHeap* const heap = ShenandoahHeap::heap();
452 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
453 const char* msg = "Concurrent remembered set scanning";
454 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
455 EventMark em("%s", msg);
456
457 ShenandoahWorkerScope scope(heap->workers(),
458 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
459 msg);
460
461 heap->try_inject_alloc_failure();
462 _generation->scan_remembered_set(true /* is_concurrent */);
463 }
464 }
465
466 void ShenandoahConcurrentGC::entry_mark_roots() {
467 ShenandoahHeap* const heap = ShenandoahHeap::heap();
468 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
469 const char* msg = "Concurrent marking roots";
470 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
471 EventMark em("%s", msg);
472
473 ShenandoahWorkerScope scope(heap->workers(),
474 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
475 "concurrent marking roots");
476
477 heap->try_inject_alloc_failure();
478 op_mark_roots();
479 }
480
481 void ShenandoahConcurrentGC::entry_mark() {
482 ShenandoahHeap* const heap = ShenandoahHeap::heap();
483 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
484 const char* msg = conc_mark_event_message();
485 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
486 EventMark em("%s", msg);
487
488 ShenandoahWorkerScope scope(heap->workers(),
489 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
490 "concurrent marking");
491
492 heap->try_inject_alloc_failure();
493 op_mark();
494 heap->try_inject_pin();
495 }
496
497 void ShenandoahConcurrentGC::entry_thread_roots() {
498 ShenandoahHeap* const heap = ShenandoahHeap::heap();
499 static const char* msg = "Concurrent thread roots";
500 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
501 EventMark em("%s", msg);
502
503 ShenandoahWorkerScope scope(heap->workers(),
504 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
505 msg);
506
507 heap->try_inject_alloc_failure();
508 heap->try_inject_pin();
509 op_thread_roots();
510 }
511
512 void ShenandoahConcurrentGC::entry_weak_refs() {
513 ShenandoahHeap* const heap = ShenandoahHeap::heap();
514 const char* msg = conc_weak_refs_event_message();
515 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
516 EventMark em("%s", msg);
517
518 ShenandoahWorkerScope scope(heap->workers(),
519 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
520 "concurrent weak references");
521
522 heap->try_inject_alloc_failure();
523 heap->try_inject_pin();
524 op_weak_refs();
525 }
526
527 void ShenandoahConcurrentGC::entry_weak_roots() {
528 ShenandoahHeap* const heap = ShenandoahHeap::heap();
529 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
530 const char* msg = conc_weak_roots_event_message();
531 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
532 EventMark em("%s", msg);
533
534 ShenandoahWorkerScope scope(heap->workers(),
535 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
536 "concurrent weak root");
537
538 heap->try_inject_alloc_failure();
539 heap->try_inject_pin();
540 op_weak_roots();
541 }
542
543 void ShenandoahConcurrentGC::entry_class_unloading() {
544 ShenandoahHeap* const heap = ShenandoahHeap::heap();
545 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
546 static const char* msg = "Concurrent class unloading";
547 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
548 EventMark em("%s", msg);
549
550 ShenandoahWorkerScope scope(heap->workers(),
551 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
552 "concurrent class unloading");
553
554 heap->try_inject_alloc_failure();
555 heap->try_inject_pin();
556 op_class_unloading();
557 }
558
559 void ShenandoahConcurrentGC::entry_strong_roots() {
560 ShenandoahHeap* const heap = ShenandoahHeap::heap();
561 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
562 static const char* msg = "Concurrent strong roots";
563 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
564 EventMark em("%s", msg);
565
566 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
567
568 ShenandoahWorkerScope scope(heap->workers(),
569 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
570 "concurrent strong root");
571
572 heap->try_inject_alloc_failure();
573 heap->try_inject_pin();
574 op_strong_roots();
575 }
576
577 void ShenandoahConcurrentGC::entry_cleanup_early() {
578 ShenandoahHeap* const heap = ShenandoahHeap::heap();
579 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
580 const char* msg = conc_cleanup_event_message();
581 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
582 EventMark em("%s", msg);
583
584 // This phase does not use workers, no need for setup
585 heap->try_inject_alloc_failure();
586 heap->try_inject_pin();
587 op_cleanup_early();
588 if (!heap->is_evacuation_in_progress()) {
589 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
590 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
591 // during promote-in-place processing.
592 heap->rebuild_free_set(true /*concurrent*/);
593 }
594 }
595
596 void ShenandoahConcurrentGC::entry_evacuate() {
597 ShenandoahHeap* const heap = ShenandoahHeap::heap();
598 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
599
600 static const char* msg = "Concurrent evacuation";
601 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
602 EventMark em("%s", msg);
603
604 ShenandoahWorkerScope scope(heap->workers(),
605 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
606 "concurrent evacuation");
607
608 heap->try_inject_alloc_failure();
609 heap->try_inject_pin();
610 op_evacuate();
611 }
612
613 void ShenandoahConcurrentGC::entry_update_thread_roots() {
614 ShenandoahHeap* const heap = ShenandoahHeap::heap();
615 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
616
617 static const char* msg = "Concurrent update thread roots";
618 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
619 EventMark em("%s", msg);
620
621 // No workers used in this phase, no setup required
622 heap->try_inject_alloc_failure();
623 heap->try_inject_pin();
624 op_update_thread_roots();
625 }
626
627 void ShenandoahConcurrentGC::entry_update_refs() {
628 ShenandoahHeap* const heap = ShenandoahHeap::heap();
629 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
630 static const char* msg = "Concurrent update references";
631 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
632 EventMark em("%s", msg);
633
634 ShenandoahWorkerScope scope(heap->workers(),
635 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
636 "concurrent reference update");
637
638 heap->try_inject_alloc_failure();
639 heap->try_inject_pin();
640 op_update_refs();
641 }
642
643 void ShenandoahConcurrentGC::entry_cleanup_complete() {
644 ShenandoahHeap* const heap = ShenandoahHeap::heap();
645 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
646 const char* msg = conc_cleanup_event_message();
647 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
648 EventMark em("%s", msg);
649
650 // This phase does not use workers, no need for setup
651 heap->try_inject_alloc_failure();
652 op_cleanup_complete();
653 }
654
655 void ShenandoahConcurrentGC::entry_reset_after_collect() {
656 ShenandoahHeap* const heap = ShenandoahHeap::heap();
657 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
658 const char* msg = conc_reset_after_collect_event_message();
659 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
660 EventMark em("%s", msg);
661
662 op_reset_after_collect();
663 }
664
665 void ShenandoahConcurrentGC::op_reset() {
666 ShenandoahHeap* const heap = ShenandoahHeap::heap();
667
668 // If it is old GC bootstrap cycle, always clear bitmap for global gen
669 // to ensure bitmap for old gen is clear for old GC cycle after this.
670 if (_do_old_gc_bootstrap) {
671 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
672 heap->global_generation()->prepare_gc();
673 } else {
674 _generation->prepare_gc();
675 }
676
677 if (heap->mode()->is_generational()) {
678 heap->old_generation()->card_scan()->mark_read_table_as_clean();
679 }
680 }
681
682 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
683 private:
684 ShenandoahMarkingContext* const _ctx;
685 public:
686 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
687
688 void heap_region_do(ShenandoahHeapRegion* r) {
689 assert(!r->has_live(), "Region %zu should have no live data", r->index());
690 if (r->is_active()) {
691 // Check if region needs updating its TAMS. We have updated it already during concurrent
692 // reset, so it is very likely we don't need to do another write here. Since most regions
693 // are not "active", this path is relatively rare.
694 if (_ctx->top_at_mark_start(r) != r->top()) {
695 _ctx->capture_top_at_mark_start(r);
696 }
697 } else {
698 assert(_ctx->top_at_mark_start(r) == r->top(),
699 "Region %zu should already have correct TAMS", r->index());
700 }
701 }
702
703 bool is_thread_safe() { return true; }
704 };
705
706 void ShenandoahConcurrentGC::start_mark() {
707 _mark.start_mark();
708 }
709
710 void ShenandoahConcurrentGC::op_init_mark() {
711 ShenandoahHeap* const heap = ShenandoahHeap::heap();
712 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
713 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
714
715 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
716 assert(!_generation->is_mark_complete(), "should not be complete");
717 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
718
719 if (heap->mode()->is_generational()) {
720 if (_generation->is_global()) {
721 heap->old_generation()->cancel_gc();
722 }
723
724 {
725 // After we swap card table below, the write-table is all clean, and the read table holds
726 // cards dirty prior to the start of GC. Young and bootstrap collection will update
727 // the write card table as a side effect of remembered set scanning. Global collection will
728 // update the card table as a side effect of global marking of old objects.
729 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
730 _generation->swap_card_tables();
731 }
732 }
733
734 if (ShenandoahVerify) {
735 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
736 heap->verifier()->verify_before_concmark(_generation);
737 }
738
739 if (VerifyBeforeGC) {
740 Universe::verify();
741 }
742
743 _generation->set_concurrent_mark_in_progress(true);
744
745 start_mark();
746
747 if (_do_old_gc_bootstrap) {
748 shenandoah_assert_generational();
749 // Update region state for both young and old regions
750 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
751 ShenandoahInitMarkUpdateRegionStateClosure cl;
752 heap->parallel_heap_region_iterate(&cl);
753 heap->old_generation()->ref_processor()->reset_thread_locals();
754 } else {
755 // Update region state for only young regions
756 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
757 ShenandoahInitMarkUpdateRegionStateClosure cl;
758 _generation->parallel_heap_region_iterate(&cl);
759 }
760
761 // Weak reference processing
762 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
763 rp->reset_thread_locals();
764
765 // Make above changes visible to worker threads
766 OrderAccess::fence();
767
768 // Arm nmethods/stack for concurrent processing
769 ShenandoahCodeRoots::arm_nmethods();
770 ShenandoahStackWatermark::change_epoch_id();
771
772 {
773 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
774 heap->propagate_gc_state_to_all_threads();
775 }
776 }
777
778 void ShenandoahConcurrentGC::op_mark_roots() {
779 _mark.mark_concurrent_roots();
780 }
781
782 void ShenandoahConcurrentGC::op_mark() {
783 _mark.concurrent_mark();
784 }
785
786 void ShenandoahConcurrentGC::op_final_mark() {
787 ShenandoahHeap* const heap = ShenandoahHeap::heap();
788 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
789 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
790
791 if (ShenandoahVerify) {
792 heap->verifier()->verify_roots_no_forwarded(_generation);
793 }
794
795 if (!heap->cancelled_gc()) {
796 _mark.finish_mark();
797 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
798
799 // Notify JVMTI that the tagmap table will need cleaning.
800 JvmtiTagMap::set_needs_cleaning();
801
802 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
803 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
804 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
805 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
806
807 // Has to be done after cset selection
808 heap->prepare_concurrent_roots();
809
810 if (!heap->collection_set()->is_empty()) {
811 LogTarget(Debug, gc, cset) lt;
812 if (lt.is_enabled()) {
813 ResourceMark rm;
814 LogStream ls(lt);
815 heap->collection_set()->print_on(&ls);
816 }
817
818 if (ShenandoahVerify) {
819 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
820 heap->verifier()->verify_before_evacuation(_generation);
821 }
822
823 heap->set_evacuation_in_progress(true);
824 // From here on, we need to update references.
825 heap->set_has_forwarded_objects(true);
826
827 // Arm nmethods/stack for concurrent processing
828 ShenandoahCodeRoots::arm_nmethods();
829 ShenandoahStackWatermark::change_epoch_id();
830
831 } else {
832 if (ShenandoahVerify) {
833 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
834 if (has_in_place_promotions(heap)) {
835 heap->verifier()->verify_after_concmark_with_promotions(_generation);
836 } else {
837 heap->verifier()->verify_after_concmark(_generation);
838 }
839 }
840 }
841 }
842
843 {
844 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
845 heap->propagate_gc_state_to_all_threads();
846 }
847 }
848
849 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
850 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
851 }
852
853 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
854 private:
855 OopClosure* const _oops;
856 public:
857 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
858
859 void do_thread(Thread* thread) override {
860 JavaThread* const jt = JavaThread::cast(thread);
861 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
862 }
863 };
864
865 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
866 private:
867 ShenandoahJavaThreadsIterator _java_threads;
868
869 public:
870 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
871 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
872 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
873 }
874
875 void work(uint worker_id) override {
876 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
877 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
878 _java_threads.threads_do(&thr_cl, worker_id);
879 }
880 };
881
882 void ShenandoahConcurrentGC::op_thread_roots() {
883 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
884 assert(heap->is_evacuation_in_progress(), "Checked by caller");
885 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
886 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
887 heap->workers()->run_task(&task);
888 }
889
890 void ShenandoahConcurrentGC::op_weak_refs() {
891 ShenandoahHeap* const heap = ShenandoahHeap::heap();
892 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
893 // Concurrent weak refs processing
894 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
895 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
896 ShenandoahBreakpoint::at_after_reference_processing_started();
897 }
898 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
899 }
900
901 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
902 private:
903 ShenandoahHeap* const _heap;
904 ShenandoahGeneration* const _generation;
905 ShenandoahMarkingContext* const _mark_context;
906 bool _evac_in_progress;
907 Thread* const _thread;
908
909 public:
910 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
911 void do_oop(oop* p);
912 void do_oop(narrowOop* p);
913 };
914
915 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
916 _heap(ShenandoahHeap::heap()),
917 _generation(generation),
918 _mark_context(ShenandoahHeap::heap()->marking_context()),
919 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
920 _thread(Thread::current()) {
921 }
922
923 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
924 const oop obj = RawAccess<>::oop_load(p);
925 if (!CompressedOops::is_null(obj)) {
926 if (!_mark_context->is_marked(obj)) {
927 if (_generation->contains(obj)) {
928 // Note: The obj is dead here. Do not touch it, just clear.
929 ShenandoahHeap::atomic_clear_oop(p, obj);
930 }
931 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
932 oop resolved = ShenandoahForwarding::get_forwardee(obj);
933 if (resolved == obj) {
934 resolved = _heap->evacuate_object(obj, _thread);
935 }
936 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
937 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
938 }
939 }
940 }
941
942 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
943 ShouldNotReachHere();
944 }
945
946 class ShenandoahIsCLDAliveClosure : public CLDClosure {
947 public:
948 void do_cld(ClassLoaderData* cld) {
949 cld->is_alive();
950 }
951 };
952
953 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
954 public:
955 void do_nmethod(nmethod* n) {
956 n->is_unloading();
957 }
958 };
959
960 // This task not only evacuates/updates marked weak roots, but also "null"
961 // dead weak roots.
962 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
963 private:
964 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
965
966 // Roots related to concurrent class unloading
967 ShenandoahClassLoaderDataRoots<true /* concurrent */>
968 _cld_roots;
969 ShenandoahConcurrentNMethodIterator _nmethod_itr;
970 ShenandoahGeneration* _generation;
971 ShenandoahPhaseTimings::Phase _phase;
972
973 public:
974 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
975 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
976 _vm_roots(phase),
977 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
978 _nmethod_itr(ShenandoahCodeRoots::table()),
979 _generation(generation),
980 _phase(phase) {}
981
982 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
983 // Notify runtime data structures of potentially dead oops
984 _vm_roots.report_num_dead();
985 }
986
987 void work(uint worker_id) override {
988 ShenandoahConcurrentWorkerSession worker_session(worker_id);
989 SuspendibleThreadSetJoiner sts_join;
990 {
991 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
992 // may race against OopStorage::release() calls.
993 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
994 _vm_roots.oops_do(&cl, worker_id);
995 }
996
997 // If we are going to perform concurrent class unloading later on, we need to
998 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
999 // can clean up immediate garbage sooner.
1000 if (ShenandoahHeap::heap()->unload_classes()) {
1001 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
1002 // CLD's holder or evacuate it.
1003 {
1004 ShenandoahIsCLDAliveClosure is_cld_alive;
1005 _cld_roots.cld_do(&is_cld_alive, worker_id);
1006 }
1007
1008 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1009 // The closure calls nmethod->is_unloading(). The is_unloading
1010 // state is cached, therefore, during concurrent class unloading phase,
1011 // we will not touch the metadata of unloading nmethods
1012 {
1013 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1014 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1015 _nmethod_itr.nmethods_do(&is_nmethod_alive);
1016 }
1017 }
1018 }
1019 };
1020
1021 void ShenandoahConcurrentGC::op_weak_roots() {
1022 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1023 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1024 {
1025 // Concurrent weak root processing
1026 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1027 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1028 heap->workers()->run_task(&task);
1029 }
1030
1031 {
1032 // It is possible for mutators executing the load reference barrier to have
1033 // loaded an oop through a weak handle that has since been nulled out by
1034 // weak root processing. Handshaking here forces them to complete the
1035 // barrier before the GC cycle continues and does something that would
1036 // change the evaluation of the barrier (for example, resetting the TAMS
1037 // on trashed regions could make an oop appear to be marked _after_ the
1038 // region has been recycled).
1039 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1040 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1041 }
1042 }
1043
1044 void ShenandoahConcurrentGC::op_class_unloading() {
1045 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1046 assert (heap->is_concurrent_weak_root_in_progress() &&
1047 heap->unload_classes(),
1048 "Checked by caller");
1049 heap->do_class_unloading();
1050 }
1051
1052 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1053 private:
1054 ShenandoahEvacuateUpdateMetadataClosure _cl;
1055
1056 public:
1057 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1058
1059 void do_nmethod(nmethod* n) {
1060 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1061 ShenandoahNMethodLocker locker(data->lock());
1062 data->oops_do(&_cl, /* fix_relocations = */ true);
1063 ShenandoahNMethod::disarm_nmethod(n);
1064 }
1065 };
1066
1067 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1068 private:
1069 ShenandoahPhaseTimings::Phase _phase;
1070 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1071 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1072 _cld_roots;
1073 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1074
1075 public:
1076 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1077 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1078 _phase(phase),
1079 _vm_roots(phase),
1080 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1081 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1082
1083 void work(uint worker_id) {
1084 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1085 {
1086 {
1087 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1088 // may race against OopStorage::release() calls.
1089 ShenandoahContextEvacuateUpdateRootsClosure cl;
1090 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1091 }
1092
1093 {
1094 ShenandoahEvacuateUpdateMetadataClosure cl;
1095 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1096 _cld_roots.cld_do(&clds, worker_id);
1097 }
1098 }
1099
1100 if (!ShenandoahHeap::heap()->unload_classes()) {
1101 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1102 ShenandoahEvacUpdateCodeCacheClosure cl;
1103 _nmethod_itr.nmethods_do(&cl);
1104 }
1105 }
1106 };
1107
1108 void ShenandoahConcurrentGC::op_strong_roots() {
1109 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1110 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1111 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1112 heap->workers()->run_task(&task);
1113 heap->set_concurrent_strong_root_in_progress(false);
1114 }
1115
1116 void ShenandoahConcurrentGC::op_cleanup_early() {
1117 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1118 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1119 "cleanup early.");
1120 ShenandoahHeap::heap()->recycle_trash();
1121 }
1122
1123 void ShenandoahConcurrentGC::op_evacuate() {
1124 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1125 }
1126
1127 void ShenandoahConcurrentGC::op_init_update_refs() {
1128 if (ShenandoahVerify) {
1129 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1130 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1131 heap->verifier()->verify_before_update_refs(_generation);
1132 }
1133 }
1134
1135 void ShenandoahConcurrentGC::op_update_refs() {
1136 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1137 }
1138
1139 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1140 private:
1141 // This closure runs when thread is stopped for handshake, which means
1142 // we can use non-concurrent closure here, as long as it only updates
1143 // locations modified by the thread itself, i.e. stack locations.
1144 ShenandoahNonConcUpdateRefsClosure _cl;
1145 public:
1146 ShenandoahUpdateThreadHandshakeClosure();
1147 void do_thread(Thread* thread) override;
1148 };
1149
1150 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1151 HandshakeClosure("Shenandoah Update Thread Roots") {
1152 }
1153
1154 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1155 if (thread->is_Java_thread()) {
1156 JavaThread* jt = JavaThread::cast(thread);
1157 ResourceMark rm;
1158 jt->oops_do(&_cl, nullptr);
1159 }
1160 }
1161
1162 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1163 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1164 // to pick up overwritten pointers that point into a cset region. If these pointers
1165 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1166 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1167 //
1168 // Therefore, at the end of update refs, we use this closure to update the thread roots
1169 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1170 // anything that has already been marked or anything that points to a region which is
1171 // not old. We do not need to worry about ABA situations where a region may become old
1172 // after the pointer is enqueued but before it is filtered. There are only two ways a
1173 // region may become old:
1174 // 1. The region is promoted in place. This is safe because such regions will never
1175 // be in the collection set. If this happens, the pointer will be preserved, essentially
1176 // becoming part of the old snapshot.
1177 // 2. The region is allocated during evacuation of old. This is also not a concern because
1178 // we haven't yet finished marking old so no mixed evacuations will happen.
1179 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1180 ShenandoahFlushSATB _flush_all_satb;
1181
1182 public:
1183 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1184 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1185 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1186 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1187 "Should be filtering pointers outside of old during old marking");
1188 }
1189
1190 void do_thread(Thread* thread) override {
1191 _update_roots.do_thread(thread);
1192 _flush_all_satb.do_thread(thread);
1193 }
1194 };
1195
1196 void ShenandoahConcurrentGC::op_update_thread_roots() {
1197 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1198 if (heap->is_concurrent_old_mark_in_progress()) {
1199 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1200 Handshake::execute(&cl);
1201 } else {
1202 ShenandoahUpdateThreadHandshakeClosure cl;
1203 Handshake::execute(&cl);
1204 }
1205 }
1206
1207 void ShenandoahConcurrentGC::op_final_update_refs() {
1208 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1209 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1210 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1211
1212 heap->finish_concurrent_roots();
1213
1214 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1215 // everything.
1216 if (heap->cancelled_gc()) {
1217 heap->clear_cancelled_gc();
1218 }
1219
1220 // Has to be done before cset is clear
1221 if (ShenandoahVerify) {
1222 heap->verifier()->verify_roots_in_to_space(_generation);
1223 }
1224
1225 // If we are running in generational mode, this will also age active regions that
1226 // haven't been used for allocation.
1227 heap->update_heap_region_states(true /*concurrent*/);
1228
1229 heap->set_update_refs_in_progress(false);
1230 heap->set_has_forwarded_objects(false);
1231
1232 if (ShenandoahVerify) {
1233 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1234 heap->verifier()->verify_after_update_refs(_generation);
1235 }
1236
1237 if (VerifyAfterGC) {
1238 Universe::verify();
1239 }
1240
1241 heap->rebuild_free_set(true /*concurrent*/);
1242 _generation->heuristics()->start_idle_span();
1243
1244 {
1245 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1246 heap->propagate_gc_state_to_all_threads();
1247 }
1248 }
1249
1250 void ShenandoahConcurrentGC::entry_final_roots() {
1251 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1252 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1253 const char* msg = conc_final_roots_event_message();
1254 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1255 EventMark em("%s", msg);
1256
1257 heap->concurrent_final_roots();
1258 }
1259
1260 void ShenandoahConcurrentGC::op_verify_final() {
1261 assert(ShenandoahVerify, "Should have been checked before");
1262 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1263 heap->verifier()->verify_after_gc(_generation);
1264 }
1265
1266 void ShenandoahConcurrentGC::op_cleanup_complete() {
1267 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1268 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1269 "cleanup complete.");
1270 ShenandoahHeap::heap()->recycle_trash();
1271 }
1272
1273 void ShenandoahConcurrentGC::op_reset_after_collect() {
1274 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1275 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1276 "reset after collection.");
1277
1278 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1279 if (heap->mode()->is_generational()) {
1280 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1281 // the young generation intact. In particular, reference processing in the old generation may potentially
1282 // need the reachability of a young generation referent of a Reference object in the old generation.
1283 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1284 heap->young_generation()->reset_mark_bitmap<false>();
1285 }
1286 } else {
1287 _generation->reset_mark_bitmap<false>();
1288 }
1289 }
1290
1291 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1292 if (ShenandoahHeap::heap()->cancelled_gc()) {
1293 _degen_point = point;
1294 return true;
1295 }
1296 return false;
1297 }
1298
1299 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1300 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1301 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1302 if (heap->unload_classes()) {
1303 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1304 } else {
1305 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1306 }
1307 }
1308
1309 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1310 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1311 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1312 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1313
1314 if (heap->unload_classes()) {
1315 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1316 } else {
1317 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1318 }
1319 }
1320
1321 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1322 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1323 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1324 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1325 if (heap->unload_classes()) {
1326 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1327 } else {
1328 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1329 }
1330 }
1331
1332 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1333 if (ShenandoahHeap::heap()->unload_classes()) {
1334 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1335 } else {
1336 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1337 }
1338 }
1339
1340 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1341 if (ShenandoahHeap::heap()->unload_classes()) {
1342 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1343 } else {
1344 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1345 }
1346 }
1347
1348 const char* ShenandoahConcurrentGC::verify_final_event_message() const {
1349 if (ShenandoahHeap::heap()->unload_classes()) {
1350 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)");
1351 } else {
1352 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
1353 }
1354 }
1355
1356 const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
1357 if (ShenandoahHeap::heap()->unload_classes()) {
1358 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
1359 } else {
1360 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
1361 }
1362 }
1363
1364 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1365 if (ShenandoahHeap::heap()->unload_classes()) {
1366 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1367 } else {
1368 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1369 }
1370 }
1371
1372 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1373 if (ShenandoahHeap::heap()->unload_classes()) {
1374 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1375 } else {
1376 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1377 }
1378 }
1379
1380 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1381 if (ShenandoahHeap::heap()->unload_classes()) {
1382 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1383 } else {
1384 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1385 }
1386 }
1387
1388 const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
1389 if (ShenandoahHeap::heap()->unload_classes()) {
1390 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
1391 } else {
1392 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
1393 }
1394 }