1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
37 #include "gc/shenandoah/shenandoahGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahLock.hpp"
40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
42 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
52 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
53 #include "memory/allocation.hpp"
54 #include "prims/jvmtiTagMap.hpp"
55 #include "runtime/vmThread.hpp"
56 #include "utilities/events.hpp"
57
58 // Breakpoint support
59 class ShenandoahBreakpointGCScope : public StackObj {
60 private:
61 const GCCause::Cause _cause;
62 public:
63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
64 if (cause == GCCause::_wb_breakpoint) {
65 ShenandoahBreakpoint::start_gc();
66 ShenandoahBreakpoint::at_before_gc();
67 }
68 }
69
70 ~ShenandoahBreakpointGCScope() {
71 if (_cause == GCCause::_wb_breakpoint) {
72 ShenandoahBreakpoint::at_after_gc();
73 }
74 }
75 };
76
77 class ShenandoahBreakpointMarkScope : public StackObj {
78 private:
79 const GCCause::Cause _cause;
80 public:
81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
82 if (_cause == GCCause::_wb_breakpoint) {
83 ShenandoahBreakpoint::at_after_marking_started();
84 }
85 }
86
87 ~ShenandoahBreakpointMarkScope() {
88 if (_cause == GCCause::_wb_breakpoint) {
89 ShenandoahBreakpoint::at_before_marking_completed();
90 }
91 }
92 };
93
94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
95 ShenandoahGC(generation),
96 _mark(generation),
97 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
98 _abbreviated(false),
99 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
100 }
101
102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
103 return _degen_point;
104 }
105
106 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
107 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
108 const char* msg = conc_init_update_refs_event_message();
109 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
110 EventMark em("%s", msg);
111
112 heap->try_inject_pin();
113 // Evacuation is complete, retire gc labs and change gc state
114 heap->concurrent_prepare_for_update_refs();
115 }
116
117 void ShenandoahConcurrentGC::entry_update_card_table() {
118 ShenandoahHeap* const heap = ShenandoahHeap::heap();
119 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
120
121 static const char* msg = "Concurrent update cards";
122 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
123 EventMark em("%s", msg);
124
125 ShenandoahWorkerScope scope(heap->workers(),
126 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
127 "concurrent update cards");
128
129 heap->try_inject_pin();
130 // Heap needs to be parsable here.
131 // Also, parallel heap region iterate must have a phase set.
132 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
133 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
134 }
135
136 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
137 ShenandoahHeap* const heap = ShenandoahHeap::heap();
138 _generation->ref_processor()->set_soft_reference_policy(
139 GCCause::should_clear_all_soft_refs(cause));
140
141 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
142
143 // Reset for upcoming marking
144 entry_reset();
145
146 // Start initial mark under STW
147 vmop_entry_init_mark();
148
149 {
150 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
151
152 // Reset task queue stats here, rather than in mark_concurrent_roots,
153 // because remembered set scan will `push` oops into the queues and
154 // resetting after this happens will lose those counts.
155 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
156
157 // Concurrent remembered set scanning
158 entry_scan_remembered_set();
159
160 // Concurrent mark roots
161 entry_mark_roots();
162 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
163 return false;
164 }
165
166 // Continue concurrent mark
167 entry_mark();
168 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
169 return false;
170 }
171 }
172
173 // Complete marking under STW, and start evacuation
174 vmop_entry_final_mark();
175
176 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
177 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
178 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
179 // from that phase.
180 if (_generation->is_concurrent_mark_in_progress()) {
181 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
182 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
183 return false;
184 }
185
186 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
187
188 // Finish all thread/stack roots if needed. This completes stack watermark processing.
189 if (heap->is_evacuation_in_progress()) {
190 entry_thread_roots();
191 }
192
193 // Process weak roots that might still point to regions that would be broken by cleanup.
194 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
195 entry_weak_refs();
196 entry_weak_roots();
197
198 // Perform concurrent class unloading before any regions get recycled. Class unloading may
199 // need to inspect unmarked objects in trashed regions.
200 if (heap->unload_classes()) {
201 entry_class_unloading();
202 }
203
204 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
205 // the space. This would be the last action if there is nothing to evacuate. Note that
206 // we will not age young-gen objects in the case that we skip evacuation.
207 entry_cleanup_early();
208
209 // Processing strong roots
210 // This may be skipped if there is nothing to update/evacuate.
211 // If so, strong_root_in_progress would be unset.
212 if (heap->is_concurrent_strong_root_in_progress()) {
213 entry_strong_roots();
214 }
215
216 // Roots processing is complete, put the weak roots flag down.
217 entry_conc_roots();
218 vmop_entry_final_roots();
219
220 // Continue the cycle with evacuation and optional update-refs.
221 // This may be skipped if there is nothing to evacuate.
222 // If so, evac_in_progress would be unset by collection set preparation code.
223 if (heap->is_evacuation_in_progress()) {
224 // Concurrently evacuate
225 entry_evacuate();
226 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
227 return false;
228 }
229
230 // Perform update-refs phase.
231 entry_concurrent_update_refs_prepare(heap);
232
233 if (ShenandoahHeap::heap()->mode()->is_generational()) {
234 entry_update_card_table();
235 }
236
237 if (ShenandoahVerify) {
238 vmop_entry_init_update_refs();
239 }
240
241 entry_update_refs();
242 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
243 return false;
244 }
245
246 // Concurrent update thread roots
247 entry_update_thread_roots();
248 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
249 return false;
250 }
251
252 vmop_entry_final_update_refs();
253
254 // Update references freed up collection set, kick the cleanup to reclaim the space.
255 entry_cleanup_complete();
256 } else {
257 _abbreviated = true;
258
259 if (heap->mode()->is_generational()) {
260 entry_complete_abbreviated_cycle();
261
262 // If the promote-in-place operation was cancelled, we can have the degenerated
263 // cycle complete the operation. It will see that no evacuations are in progress,
264 // and that there are regions wanting promotion. The risk with not handling the
265 // cancellation would be failing to restore top for these regions and leaving
266 // them unable to serve allocations for the old generation.
267 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
268 return false;
269 }
270 }
271
272 // In normal cycle, final-update-refs would verify at the end of the cycle.
273 // In abbreviated cycle, we need to verify separately.
274 if (ShenandoahVerify) {
275 vmop_entry_final_verify();
276 }
277 }
278
279 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
280 // abbreviated cycle.
281 if (heap->mode()->is_generational()) {
282 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
283 }
284
285 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
286 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
287 // reducing the likelihood that GC will degenerate.
288 entry_reset_after_collect();
289
290 return true;
291 }
292
293 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
294 shenandoah_assert_generational();
295
296 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
297
298 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
299 static const char* msg = "Concurrent complete abbreviated cycle";
300 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
301 EventMark em("%s", msg);
302
303 ShenandoahWorkerScope scope(heap->workers(),
304 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
305 msg);
306
307 heap->try_inject_pin();
308 // We chose not to evacuate because we found sufficient immediate garbage.
309 // However, there may still be regions to promote in place, so do that now.
310 if (heap->old_generation()->has_in_place_promotions()) {
311 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
312 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
313 heap->promote_regions_in_place(_generation, true);
314 }
315
316 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
317 // the control thread will detect it on its next iteration and run a degenerated young cycle.
318 if (!heap->cancelled_gc() && !_generation->is_old()) {
319 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
320 heap->update_region_ages(_generation->complete_marking_context());
321 }
322 }
323
324 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
325 ShenandoahHeap* const heap = ShenandoahHeap::heap();
326 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
327 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
328
329 heap->try_inject_alloc_failure();
330 VM_ShenandoahInitMark op(this);
331 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
332 }
333
334 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
335 ShenandoahHeap* const heap = ShenandoahHeap::heap();
336 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
337 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
338
339 heap->try_inject_alloc_failure();
340 VM_ShenandoahFinalMarkStartEvac op(this);
341 VMThread::execute(&op); // jump to entry_final_mark under safepoint
342 heap->try_inject_pin();
343 }
344
345 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
346 ShenandoahHeap* const heap = ShenandoahHeap::heap();
347 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
348 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
349
350 heap->try_inject_alloc_failure();
351 heap->try_inject_pin();
352 VM_ShenandoahInitUpdateRefs op(this);
353 VMThread::execute(&op);
354 }
355
356 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
357 ShenandoahHeap* const heap = ShenandoahHeap::heap();
358 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
359 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
360
361 heap->try_inject_alloc_failure();
362 heap->try_inject_pin();
363 VM_ShenandoahFinalUpdateRefs op(this);
364 VMThread::execute(&op);
365 }
366
367 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
368 ShenandoahHeap* const heap = ShenandoahHeap::heap();
369 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
370 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
371
372 // This phase does not use workers, no need for setup
373 heap->try_inject_alloc_failure();
374 VM_ShenandoahFinalRoots op(this);
375 VMThread::execute(&op);
376 }
377
378 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
379 ShenandoahHeap* const heap = ShenandoahHeap::heap();
380 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
381 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
382
383 // This phase does not use workers, no need for setup
384 heap->try_inject_alloc_failure();
385 heap->try_inject_pin();
386 VM_ShenandoahFinalVerify op(this);
387 VMThread::execute(&op);
388 }
389
390 void ShenandoahConcurrentGC::entry_init_mark() {
391 const char* msg = init_mark_event_message();
392 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
393 EventMark em("%s", msg);
394
395 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
396 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
397 "init marking");
398
399 op_init_mark();
400 }
401
402 void ShenandoahConcurrentGC::entry_final_mark() {
403 const char* msg = final_mark_event_message();
404 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
405 EventMark em("%s", msg);
406
407 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
408 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
409 "final marking");
410
411 op_final_mark();
412 }
413
414 void ShenandoahConcurrentGC::entry_init_update_refs() {
415 static const char* msg = "Pause Init Update Refs";
416 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
417 EventMark em("%s", msg);
418
419 // No workers used in this phase, no setup required
420 op_init_update_refs();
421 }
422
423 void ShenandoahConcurrentGC::entry_final_update_refs() {
424 static const char* msg = "Pause Final Update Refs";
425 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
426 EventMark em("%s", msg);
427
428 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
429 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
430 "final reference update");
431
432 op_final_update_refs();
433 }
434
435 void ShenandoahConcurrentGC::entry_final_verify() {
436 const char* msg = verify_final_event_message();
437 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
438 EventMark em("%s", msg);
439
440 op_verify_final();
441 }
442
443 void ShenandoahConcurrentGC::entry_reset() {
444 ShenandoahHeap* const heap = ShenandoahHeap::heap();
445 heap->release_injected_pins();
446 heap->try_inject_alloc_failure();
447
448 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
449 {
450 const char* msg = conc_reset_event_message();
451 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
452 EventMark em("%s", msg);
453
454 ShenandoahWorkerScope scope(heap->workers(),
455 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
456 msg);
457 op_reset();
458 }
459 }
460
461 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
462 if (_generation->is_young()) {
463 ShenandoahHeap* const heap = ShenandoahHeap::heap();
464 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
465 const char* msg = "Concurrent remembered set scanning";
466 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
467 EventMark em("%s", msg);
468
469 ShenandoahWorkerScope scope(heap->workers(),
470 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
471 msg);
472
473 heap->try_inject_alloc_failure();
474 _generation->scan_remembered_set(true /* is_concurrent */);
475 }
476 }
477
478 void ShenandoahConcurrentGC::entry_mark_roots() {
479 ShenandoahHeap* const heap = ShenandoahHeap::heap();
480 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
481 const char* msg = "Concurrent marking roots";
482 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
483 EventMark em("%s", msg);
484
485 ShenandoahWorkerScope scope(heap->workers(),
486 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
487 "concurrent marking roots");
488
489 heap->try_inject_alloc_failure();
490 op_mark_roots();
491 }
492
493 void ShenandoahConcurrentGC::entry_mark() {
494 ShenandoahHeap* const heap = ShenandoahHeap::heap();
495 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
496 const char* msg = conc_mark_event_message();
497 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
498 EventMark em("%s", msg);
499
500 ShenandoahWorkerScope scope(heap->workers(),
501 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
502 "concurrent marking");
503
504 heap->try_inject_alloc_failure();
505 op_mark();
506 heap->try_inject_pin();
507 }
508
509 void ShenandoahConcurrentGC::entry_thread_roots() {
510 ShenandoahHeap* const heap = ShenandoahHeap::heap();
511 static const char* msg = "Concurrent thread roots";
512 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
513 EventMark em("%s", msg);
514
515 ShenandoahWorkerScope scope(heap->workers(),
516 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
517 msg);
518
519 heap->try_inject_alloc_failure();
520 heap->try_inject_pin();
521 op_thread_roots();
522 }
523
524 void ShenandoahConcurrentGC::entry_weak_refs() {
525 ShenandoahHeap* const heap = ShenandoahHeap::heap();
526 const char* msg = conc_weak_refs_event_message();
527 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
528 EventMark em("%s", msg);
529
530 ShenandoahWorkerScope scope(heap->workers(),
531 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
532 "concurrent weak references");
533
534 heap->try_inject_alloc_failure();
535 heap->try_inject_pin();
536 op_weak_refs();
537 }
538
539 void ShenandoahConcurrentGC::entry_weak_roots() {
540 ShenandoahHeap* const heap = ShenandoahHeap::heap();
541 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
542 const char* msg = conc_weak_roots_event_message();
543 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
544 EventMark em("%s", msg);
545
546 ShenandoahWorkerScope scope(heap->workers(),
547 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
548 "concurrent weak root");
549
550 heap->try_inject_alloc_failure();
551 heap->try_inject_pin();
552 op_weak_roots();
553 }
554
555 void ShenandoahConcurrentGC::entry_class_unloading() {
556 ShenandoahHeap* const heap = ShenandoahHeap::heap();
557 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
558 static const char* msg = "Concurrent class unloading";
559 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
560 EventMark em("%s", msg);
561
562 ShenandoahWorkerScope scope(heap->workers(),
563 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
564 "concurrent class unloading");
565
566 heap->try_inject_alloc_failure();
567 heap->try_inject_pin();
568 op_class_unloading();
569 }
570
571 void ShenandoahConcurrentGC::entry_strong_roots() {
572 ShenandoahHeap* const heap = ShenandoahHeap::heap();
573 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
574 static const char* msg = "Concurrent strong roots";
575 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
576 EventMark em("%s", msg);
577
578 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
579
580 ShenandoahWorkerScope scope(heap->workers(),
581 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
582 "concurrent strong root");
583
584 heap->try_inject_alloc_failure();
585 heap->try_inject_pin();
586 op_strong_roots();
587 }
588
589 void ShenandoahConcurrentGC::entry_cleanup_early() {
590 ShenandoahHeap* const heap = ShenandoahHeap::heap();
591 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
592 const char* msg = conc_cleanup_event_message();
593 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
594 EventMark em("%s", msg);
595
596 // This phase does not use workers, no need for setup
597 heap->try_inject_alloc_failure();
598 heap->try_inject_pin();
599 op_cleanup_early();
600 if (!heap->is_evacuation_in_progress()) {
601 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
602 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
603 // during promote-in-place processing.
604 heap->rebuild_free_set(true /*concurrent*/);
605 }
606 }
607
608 void ShenandoahConcurrentGC::entry_evacuate() {
609 ShenandoahHeap* const heap = ShenandoahHeap::heap();
610 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
611
612 static const char* msg = "Concurrent evacuation";
613 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
614 EventMark em("%s", msg);
615
616 ShenandoahWorkerScope scope(heap->workers(),
617 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
618 "concurrent evacuation");
619
620 heap->try_inject_alloc_failure();
621 heap->try_inject_pin();
622 op_evacuate();
623 }
624
625 void ShenandoahConcurrentGC::entry_update_thread_roots() {
626 ShenandoahHeap* const heap = ShenandoahHeap::heap();
627 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
628
629 static const char* msg = "Concurrent update thread roots";
630 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
631 EventMark em("%s", msg);
632
633 // No workers used in this phase, no setup required
634 heap->try_inject_alloc_failure();
635 heap->try_inject_pin();
636 op_update_thread_roots();
637 }
638
639 void ShenandoahConcurrentGC::entry_update_refs() {
640 ShenandoahHeap* const heap = ShenandoahHeap::heap();
641 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
642 static const char* msg = "Concurrent update references";
643 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
644 EventMark em("%s", msg);
645
646 ShenandoahWorkerScope scope(heap->workers(),
647 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
648 "concurrent reference update");
649
650 heap->try_inject_alloc_failure();
651 heap->try_inject_pin();
652 op_update_refs();
653 }
654
655 void ShenandoahConcurrentGC::entry_cleanup_complete() {
656 ShenandoahHeap* const heap = ShenandoahHeap::heap();
657 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
658 const char* msg = conc_cleanup_event_message();
659 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
660 EventMark em("%s", msg);
661
662 // This phase does not use workers, no need for setup
663 heap->try_inject_alloc_failure();
664 op_cleanup_complete();
665 }
666
667 void ShenandoahConcurrentGC::entry_reset_after_collect() {
668 ShenandoahHeap* const heap = ShenandoahHeap::heap();
669 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
670 const char* msg = conc_reset_after_collect_event_message();
671 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
672 EventMark em("%s", msg);
673
674 op_reset_after_collect();
675 }
676
677 void ShenandoahConcurrentGC::op_reset() {
678 ShenandoahHeap* const heap = ShenandoahHeap::heap();
679
680 // If it is old GC bootstrap cycle, always clear bitmap for global gen
681 // to ensure bitmap for old gen is clear for old GC cycle after this.
682 if (_do_old_gc_bootstrap) {
683 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
684 heap->global_generation()->prepare_gc();
685 } else {
686 _generation->prepare_gc();
687 }
688
689 if (heap->mode()->is_generational()) {
690 heap->old_generation()->card_scan()->mark_read_table_as_clean();
691 }
692 }
693
694 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
695 private:
696 ShenandoahMarkingContext* const _ctx;
697 public:
698 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
699
700 void heap_region_do(ShenandoahHeapRegion* r) {
701 assert(!r->has_live(), "Region %zu should have no live data", r->index());
702 if (r->is_active()) {
703 // Check if region needs updating its TAMS. We have updated it already during concurrent
704 // reset, so it is very likely we don't need to do another write here. Since most regions
705 // are not "active", this path is relatively rare.
706 if (_ctx->top_at_mark_start(r) != r->top()) {
707 _ctx->capture_top_at_mark_start(r);
708 }
709 } else {
710 assert(_ctx->top_at_mark_start(r) == r->top(),
711 "Region %zu should already have correct TAMS", r->index());
712 }
713 }
714
715 bool is_thread_safe() { return true; }
716 };
717
718 void ShenandoahConcurrentGC::start_mark() {
719 _mark.start_mark();
720 }
721
722 void ShenandoahConcurrentGC::op_init_mark() {
723 ShenandoahHeap* const heap = ShenandoahHeap::heap();
724 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
725 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
726
727 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
728 assert(!_generation->is_mark_complete(), "should not be complete");
729 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
730
731 if (heap->mode()->is_generational()) {
732 if (_generation->is_global()) {
733 heap->old_generation()->cancel_gc();
734 }
735
736 {
737 // After we swap card table below, the write-table is all clean, and the read table holds
738 // cards dirty prior to the start of GC. Young and bootstrap collection will update
739 // the write card table as a side effect of remembered set scanning. Global collection will
740 // update the card table as a side effect of global marking of old objects.
741 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
742 _generation->swap_card_tables();
743 }
744 }
745
746 if (ShenandoahVerify) {
747 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
748 heap->verifier()->verify_before_concmark(_generation);
749 }
750
751 if (VerifyBeforeGC) {
752 Universe::verify();
753 }
754
755 _generation->set_concurrent_mark_in_progress(true);
756
757 start_mark();
758
759 if (_do_old_gc_bootstrap) {
760 shenandoah_assert_generational();
761 // Update region state for both young and old regions
762 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
763 ShenandoahInitMarkUpdateRegionStateClosure cl;
764 heap->parallel_heap_region_iterate(&cl);
765 heap->old_generation()->ref_processor()->reset_thread_locals();
766 } else {
767 // Update region state for only young regions
768 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
769 ShenandoahInitMarkUpdateRegionStateClosure cl;
770 _generation->parallel_heap_region_iterate(&cl);
771 }
772
773 // Weak reference processing
774 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
775 rp->reset_thread_locals();
776
777 // Make above changes visible to worker threads
778 OrderAccess::fence();
779
780 // Arm nmethods/stack for concurrent processing
781 ShenandoahCodeRoots::arm_nmethods();
782 ShenandoahStackWatermark::change_epoch_id();
783
784 {
785 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
786 heap->propagate_gc_state_to_all_threads();
787 }
788 }
789
790 void ShenandoahConcurrentGC::op_mark_roots() {
791 _mark.mark_concurrent_roots();
792 }
793
794 void ShenandoahConcurrentGC::op_mark() {
795 _mark.concurrent_mark();
796 }
797
798 void ShenandoahConcurrentGC::op_final_mark() {
799 ShenandoahHeap* const heap = ShenandoahHeap::heap();
800 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
801 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
802
803 if (ShenandoahVerify) {
804 heap->verifier()->verify_roots_no_forwarded(_generation);
805 }
806
807 if (!heap->cancelled_gc()) {
808 _mark.finish_mark();
809 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
810
811 // Notify JVMTI that the tagmap table will need cleaning.
812 JvmtiTagMap::set_needs_cleaning();
813
814 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
815 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
816 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
817 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
818
819 // Has to be done after cset selection
820 heap->prepare_concurrent_roots();
821
822 if (!heap->collection_set()->is_empty()) {
823 LogTarget(Debug, gc, cset) lt;
824 if (lt.is_enabled()) {
825 ResourceMark rm;
826 LogStream ls(lt);
827 heap->collection_set()->print_on(&ls);
828 }
829
830 if (ShenandoahVerify) {
831 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
832 heap->verifier()->verify_before_evacuation(_generation);
833 }
834
835 heap->set_evacuation_in_progress(true);
836 // From here on, we need to update references.
837 heap->set_has_forwarded_objects(true);
838 } else {
839 if (ShenandoahVerify) {
840 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
841 if (has_in_place_promotions(heap)) {
842 heap->verifier()->verify_after_concmark_with_promotions(_generation);
843 } else {
844 heap->verifier()->verify_after_concmark(_generation);
845 }
846 }
847 }
848 }
849
850 // Arm nmethods/stack for concurrent processing
851 ShenandoahCodeRoots::arm_nmethods();
852 ShenandoahStackWatermark::change_epoch_id();
853
854 {
855 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
856 heap->propagate_gc_state_to_all_threads();
857 }
858 }
859
860 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
861 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
862 }
863
864 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
865 private:
866 OopClosure* const _oops;
867 public:
868 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
869
870 void do_thread(Thread* thread) override {
871 JavaThread* const jt = JavaThread::cast(thread);
872 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
873 }
874 };
875
876 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
877 private:
878 ShenandoahJavaThreadsIterator _java_threads;
879
880 public:
881 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
882 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
883 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
884 }
885
886 void work(uint worker_id) override {
887 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
888 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
889 _java_threads.threads_do(&thr_cl, worker_id);
890 }
891 };
892
893 void ShenandoahConcurrentGC::op_thread_roots() {
894 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
895 assert(heap->is_evacuation_in_progress(), "Checked by caller");
896 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
897 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
898 heap->workers()->run_task(&task);
899 }
900
901 void ShenandoahConcurrentGC::op_weak_refs() {
902 ShenandoahHeap* const heap = ShenandoahHeap::heap();
903 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
904 // Concurrent weak refs processing
905 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
906 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
907 ShenandoahBreakpoint::at_after_reference_processing_started();
908 }
909 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
910 }
911
912 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
913 private:
914 ShenandoahHeap* const _heap;
915 ShenandoahGeneration* const _generation;
916 ShenandoahMarkingContext* const _mark_context;
917 bool _evac_in_progress;
918 Thread* const _thread;
919
920 public:
921 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
922 void do_oop(oop* p);
923 void do_oop(narrowOop* p);
924 };
925
926 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
927 _heap(ShenandoahHeap::heap()),
928 _generation(generation),
929 _mark_context(ShenandoahHeap::heap()->marking_context()),
930 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
931 _thread(Thread::current()) {
932 }
933
934 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
935 const oop obj = RawAccess<>::oop_load(p);
936 if (!CompressedOops::is_null(obj)) {
937 if (!_mark_context->is_marked(obj)) {
938 if (_generation->contains(obj)) {
939 // Note: The obj is dead here. Do not touch it, just clear.
940 ShenandoahHeap::atomic_clear_oop(p, obj);
941 }
942 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
943 oop resolved = ShenandoahForwarding::get_forwardee(obj);
944 if (resolved == obj) {
945 resolved = _heap->evacuate_object(obj, _thread);
946 }
947 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
948 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
949 }
950 }
951 }
952
953 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
954 ShouldNotReachHere();
955 }
956
957 class ShenandoahIsCLDAliveClosure : public CLDClosure {
958 public:
959 void do_cld(ClassLoaderData* cld) {
960 cld->is_alive();
961 }
962 };
963
964 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
965 public:
966 void do_nmethod(nmethod* n) {
967 n->is_unloading();
968 }
969 };
970
971 // This task not only evacuates/updates marked weak roots, but also "null"
972 // dead weak roots.
973 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
974 private:
975 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
976
977 // Roots related to concurrent class unloading
978 ShenandoahClassLoaderDataRoots<true /* concurrent */>
979 _cld_roots;
980 ShenandoahConcurrentNMethodIterator _nmethod_itr;
981 ShenandoahGeneration* _generation;
982 ShenandoahPhaseTimings::Phase _phase;
983
984 public:
985 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
986 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
987 _vm_roots(phase),
988 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
989 _nmethod_itr(ShenandoahCodeRoots::table()),
990 _generation(generation),
991 _phase(phase) {}
992
993 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
994 // Notify runtime data structures of potentially dead oops
995 _vm_roots.report_num_dead();
996 }
997
998 void work(uint worker_id) override {
999 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1000 SuspendibleThreadSetJoiner sts_join;
1001 {
1002 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1003 // may race against OopStorage::release() calls.
1004 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
1005 _vm_roots.oops_do(&cl, worker_id);
1006 }
1007
1008 // If we are going to perform concurrent class unloading later on, we need to
1009 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
1010 // can clean up immediate garbage sooner.
1011 if (ShenandoahHeap::heap()->unload_classes()) {
1012 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
1013 // CLD's holder or evacuate it.
1014 {
1015 ShenandoahIsCLDAliveClosure is_cld_alive;
1016 _cld_roots.cld_do(&is_cld_alive, worker_id);
1017 }
1018
1019 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1020 // The closure calls nmethod->is_unloading(). The is_unloading
1021 // state is cached, therefore, during concurrent class unloading phase,
1022 // we will not touch the metadata of unloading nmethods
1023 {
1024 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1025 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1026 _nmethod_itr.nmethods_do(&is_nmethod_alive);
1027 }
1028 }
1029 }
1030 };
1031
1032 void ShenandoahConcurrentGC::op_weak_roots() {
1033 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1034 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1035 {
1036 // Concurrent weak root processing
1037 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1038 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1039 heap->workers()->run_task(&task);
1040 }
1041
1042 {
1043 // It is possible for mutators executing the load reference barrier to have
1044 // loaded an oop through a weak handle that has since been nulled out by
1045 // weak root processing. Handshaking here forces them to complete the
1046 // barrier before the GC cycle continues and does something that would
1047 // change the evaluation of the barrier (for example, resetting the TAMS
1048 // on trashed regions could make an oop appear to be marked _after_ the
1049 // region has been recycled).
1050 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1051 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1052 }
1053 }
1054
1055 void ShenandoahConcurrentGC::op_class_unloading() {
1056 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1057 assert (heap->is_concurrent_weak_root_in_progress() &&
1058 heap->unload_classes(),
1059 "Checked by caller");
1060 heap->do_class_unloading();
1061 }
1062
1063 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1064 private:
1065 ShenandoahEvacuateUpdateMetadataClosure _cl;
1066
1067 public:
1068 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1069
1070 void do_nmethod(nmethod* n) {
1071 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1072 ShenandoahNMethodLocker locker(data->lock());
1073 data->oops_do(&_cl, /* fix_relocations = */ true);
1074 }
1075 };
1076
1077 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1078 private:
1079 ShenandoahPhaseTimings::Phase _phase;
1080 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1081 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1082 _cld_roots;
1083 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1084
1085 public:
1086 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1087 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1088 _phase(phase),
1089 _vm_roots(phase),
1090 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1091 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1092
1093 void work(uint worker_id) {
1094 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1095 {
1096 {
1097 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1098 // may race against OopStorage::release() calls.
1099 ShenandoahContextEvacuateUpdateRootsClosure cl;
1100 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1101 }
1102
1103 {
1104 ShenandoahEvacuateUpdateMetadataClosure cl;
1105 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1106 _cld_roots.cld_do(&clds, worker_id);
1107 }
1108 }
1109
1110 if (!ShenandoahHeap::heap()->unload_classes()) {
1111 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1112 ShenandoahEvacUpdateCodeCacheClosure cl;
1113 _nmethod_itr.nmethods_do(&cl);
1114 }
1115 }
1116 };
1117
1118 void ShenandoahConcurrentGC::op_strong_roots() {
1119 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1120 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1121 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1122 heap->workers()->run_task(&task);
1123 heap->set_concurrent_strong_root_in_progress(false);
1124 }
1125
1126 void ShenandoahConcurrentGC::op_cleanup_early() {
1127 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1128 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1129 "cleanup early.");
1130 ShenandoahHeap::heap()->recycle_trash();
1131 }
1132
1133 void ShenandoahConcurrentGC::op_evacuate() {
1134 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1135 }
1136
1137 void ShenandoahConcurrentGC::op_init_update_refs() {
1138 if (ShenandoahVerify) {
1139 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1140 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1141 heap->verifier()->verify_before_update_refs(_generation);
1142 }
1143 }
1144
1145 void ShenandoahConcurrentGC::op_update_refs() {
1146 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1147 }
1148
1149 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1150 private:
1151 // This closure runs when thread is stopped for handshake, which means
1152 // we can use non-concurrent closure here, as long as it only updates
1153 // locations modified by the thread itself, i.e. stack locations.
1154 ShenandoahNonConcUpdateRefsClosure _cl;
1155 public:
1156 ShenandoahUpdateThreadHandshakeClosure();
1157 void do_thread(Thread* thread) override;
1158 };
1159
1160 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1161 HandshakeClosure("Shenandoah Update Thread Roots") {
1162 }
1163
1164 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1165 if (thread->is_Java_thread()) {
1166 JavaThread* jt = JavaThread::cast(thread);
1167 ResourceMark rm;
1168 jt->oops_do(&_cl, nullptr);
1169 }
1170 }
1171
1172 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1173 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1174 // to pick up overwritten pointers that point into a cset region. If these pointers
1175 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1176 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1177 //
1178 // Therefore, at the end of update refs, we use this closure to update the thread roots
1179 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1180 // anything that has already been marked or anything that points to a region which is
1181 // not old. We do not need to worry about ABA situations where a region may become old
1182 // after the pointer is enqueued but before it is filtered. There are only two ways a
1183 // region may become old:
1184 // 1. The region is promoted in place. This is safe because such regions will never
1185 // be in the collection set. If this happens, the pointer will be preserved, essentially
1186 // becoming part of the old snapshot.
1187 // 2. The region is allocated during evacuation of old. This is also not a concern because
1188 // we haven't yet finished marking old so no mixed evacuations will happen.
1189 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1190 ShenandoahFlushSATB _flush_all_satb;
1191
1192 public:
1193 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1194 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1195 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1196 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1197 "Should be filtering pointers outside of old during old marking");
1198 }
1199
1200 void do_thread(Thread* thread) override {
1201 _update_roots.do_thread(thread);
1202 _flush_all_satb.do_thread(thread);
1203 }
1204 };
1205
1206 void ShenandoahConcurrentGC::op_update_thread_roots() {
1207 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1208 if (heap->is_concurrent_old_mark_in_progress()) {
1209 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1210 Handshake::execute(&cl);
1211 } else {
1212 ShenandoahUpdateThreadHandshakeClosure cl;
1213 Handshake::execute(&cl);
1214 }
1215 }
1216
1217 void ShenandoahConcurrentGC::op_final_update_refs() {
1218 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1219 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1220 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1221
1222 heap->finish_concurrent_roots();
1223
1224 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1225 // everything.
1226 if (heap->cancelled_gc()) {
1227 heap->clear_cancelled_gc();
1228 }
1229
1230 // Has to be done before cset is clear
1231 if (ShenandoahVerify) {
1232 heap->verifier()->verify_roots_in_to_space(_generation);
1233 }
1234
1235 // If we are running in generational mode, this will also age active regions that
1236 // haven't been used for allocation.
1237 heap->update_heap_region_states(true /*concurrent*/);
1238
1239 heap->set_update_refs_in_progress(false);
1240 heap->set_has_forwarded_objects(false);
1241
1242 if (ShenandoahVerify) {
1243 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1244 heap->verifier()->verify_after_update_refs(_generation);
1245 }
1246
1247 if (VerifyAfterGC) {
1248 Universe::verify();
1249 }
1250
1251 heap->rebuild_free_set(true /*concurrent*/);
1252 _generation->heuristics()->start_idle_span();
1253
1254 {
1255 // Final pause: update GC barriers to idle state.
1256 ShenandoahCodeRoots::arm_nmethods();
1257 ShenandoahStackWatermark::change_epoch_id();
1258 }
1259
1260 {
1261 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1262 heap->propagate_gc_state_to_all_threads();
1263 }
1264 }
1265
1266 void ShenandoahConcurrentGC::entry_conc_roots() {
1267 const char* msg = conc_final_roots_event_message();
1268 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_roots);
1269 EventMark em("%s", msg);
1270
1271 ShenandoahHeap::heap()->op_conc_roots();
1272 }
1273
1274 void ShenandoahConcurrentGC::entry_final_roots() {
1275 const char* msg = "Pause Final Roots";
1276 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1277 EventMark em("%s", msg);
1278
1279 ShenandoahHeap::heap()->op_final_roots();
1280 }
1281
1282 void ShenandoahConcurrentGC::op_verify_final() {
1283 assert(ShenandoahVerify, "Should have been checked before");
1284 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1285 heap->verifier()->verify_after_gc(_generation);
1286 }
1287
1288 void ShenandoahConcurrentGC::op_cleanup_complete() {
1289 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1290 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1291 "cleanup complete.");
1292 ShenandoahHeap::heap()->recycle_trash();
1293 }
1294
1295 void ShenandoahConcurrentGC::op_reset_after_collect() {
1296 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1297 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1298 "reset after collection.");
1299
1300 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1301 if (heap->mode()->is_generational()) {
1302 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1303 // the young generation intact. In particular, reference processing in the old generation may potentially
1304 // need the reachability of a young generation referent of a Reference object in the old generation.
1305 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1306 heap->young_generation()->reset_mark_bitmap<false>();
1307 }
1308 } else {
1309 _generation->reset_mark_bitmap<false>();
1310 }
1311 }
1312
1313 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1314 if (ShenandoahHeap::heap()->cancelled_gc()) {
1315 _degen_point = point;
1316 return true;
1317 }
1318 return false;
1319 }
1320
1321 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1322 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1323 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1324 if (heap->unload_classes()) {
1325 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1326 } else {
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1328 }
1329 }
1330
1331 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1332 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1333 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1334 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1335
1336 if (heap->unload_classes()) {
1337 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1338 } else {
1339 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1340 }
1341 }
1342
1343 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1344 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1345 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1346 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1347 if (heap->unload_classes()) {
1348 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1349 } else {
1350 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1351 }
1352 }
1353
1354 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1355 if (ShenandoahHeap::heap()->unload_classes()) {
1356 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1357 } else {
1358 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1359 }
1360 }
1361
1362 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1363 if (ShenandoahHeap::heap()->unload_classes()) {
1364 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1365 } else {
1366 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1367 }
1368 }
1369
1370 const char* ShenandoahConcurrentGC::verify_final_event_message() const {
1371 if (ShenandoahHeap::heap()->unload_classes()) {
1372 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)");
1373 } else {
1374 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
1375 }
1376 }
1377
1378 const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
1379 if (ShenandoahHeap::heap()->unload_classes()) {
1380 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
1381 } else {
1382 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
1383 }
1384 }
1385
1386 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1387 if (ShenandoahHeap::heap()->unload_classes()) {
1388 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1389 } else {
1390 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1391 }
1392 }
1393
1394 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1395 if (ShenandoahHeap::heap()->unload_classes()) {
1396 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1397 } else {
1398 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1399 }
1400 }
1401
1402 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1403 if (ShenandoahHeap::heap()->unload_classes()) {
1404 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1405 } else {
1406 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1407 }
1408 }
1409
1410 const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
1411 if (ShenandoahHeap::heap()->unload_classes()) {
1412 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
1413 } else {
1414 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
1415 }
1416 }