1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
37 #include "gc/shenandoah/shenandoahGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahLock.hpp"
40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
42 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
52 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
53 #include "memory/allocation.hpp"
54 #include "prims/jvmtiTagMap.hpp"
55 #include "runtime/vmThread.hpp"
56 #include "utilities/events.hpp"
57
58 // Breakpoint support
59 class ShenandoahBreakpointGCScope : public StackObj {
60 private:
61 const GCCause::Cause _cause;
62 public:
63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
64 if (cause == GCCause::_wb_breakpoint) {
65 ShenandoahBreakpoint::start_gc();
66 ShenandoahBreakpoint::at_before_gc();
67 }
68 }
69
70 ~ShenandoahBreakpointGCScope() {
71 if (_cause == GCCause::_wb_breakpoint) {
72 ShenandoahBreakpoint::at_after_gc();
73 }
74 }
75 };
76
77 class ShenandoahBreakpointMarkScope : public StackObj {
78 private:
79 const GCCause::Cause _cause;
80 public:
81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
82 if (_cause == GCCause::_wb_breakpoint) {
83 ShenandoahBreakpoint::at_after_marking_started();
84 }
85 }
86
87 ~ShenandoahBreakpointMarkScope() {
88 if (_cause == GCCause::_wb_breakpoint) {
89 ShenandoahBreakpoint::at_before_marking_completed();
90 }
91 }
92 };
93
94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
95 ShenandoahGC(generation),
96 _mark(generation),
97 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
98 _abbreviated(false),
99 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
100 }
101
102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
103 return _degen_point;
104 }
105
106 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
107 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
108 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent init update refs", "");
109 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
110 EventMark em("%s", msg);
111
112 heap->try_inject_pin();
113 // Evacuation is complete, retire gc labs and change gc state
114 heap->concurrent_prepare_for_update_refs();
115 }
116
117 void ShenandoahConcurrentGC::entry_update_card_table() {
118 ShenandoahHeap* const heap = ShenandoahHeap::heap();
119 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
120 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update cards", "");
121 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
122 EventMark em("%s", msg);
123
124 ShenandoahWorkerScope scope(heap->workers(),
125 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
126 "concurrent update cards");
127
128 heap->try_inject_pin();
129 // Heap needs to be parsable here.
130 // Also, parallel heap region iterate must have a phase set.
131 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
132 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
133 }
134
135 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
136 ShenandoahHeap* const heap = ShenandoahHeap::heap();
137 _generation->ref_processor()->set_soft_reference_policy(
138 GCCause::should_clear_all_soft_refs(cause));
139
140 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent GC", "");
141 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_gc, /* log_heap_usage = */ true);
142
143 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
144
145 // Reset for upcoming marking
146 entry_reset();
147
148 // Start initial mark under STW
149 vmop_entry_init_mark();
150
151 {
152 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
153
154 // Reset task queue stats here, rather than in mark_concurrent_roots,
155 // because remembered set scan will `push` oops into the queues and
156 // resetting after this happens will lose those counts.
157 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
158
159 // Concurrent remembered set scanning
160 entry_scan_remembered_set();
161
162 // Concurrent mark roots
163 entry_mark_roots();
164 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
165 return false;
166 }
167
168 // Continue concurrent mark
169 entry_mark();
170 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
171 return false;
172 }
173 }
174
175 // Complete marking under STW, and start evacuation
176 vmop_entry_final_mark();
177
178 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
179 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
180 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
181 // from that phase.
182 if (_generation->is_concurrent_mark_in_progress()) {
183 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
184 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
185 return false;
186 }
187
188 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
189
190 // Finish all thread/stack roots if needed. This completes stack watermark processing.
191 if (heap->is_evacuation_in_progress()) {
192 entry_thread_roots();
193 }
194
195 // Process weak roots that might still point to regions that would be broken by cleanup.
196 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
197 entry_weak_refs();
198 entry_weak_roots();
199
200 // Perform concurrent class unloading before any regions get recycled. Class unloading may
201 // need to inspect unmarked objects in trashed regions.
202 if (heap->unload_classes()) {
203 entry_class_unloading();
204 }
205
206 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
207 // the space. This would be the last action if there is nothing to evacuate. Note that
208 // we will not age young-gen objects in the case that we skip evacuation.
209 entry_cleanup_early();
210
211 // Processing strong roots
212 // This may be skipped if there is nothing to update/evacuate.
213 // If so, strong_root_in_progress would be unset.
214 if (heap->is_concurrent_strong_root_in_progress()) {
215 entry_strong_roots();
216 }
217
218 // Roots processing is complete, put the weak roots flag down.
219 vmop_entry_final_roots();
220
221 // Continue the cycle with evacuation and optional update-refs.
222 // This may be skipped if there is nothing to evacuate.
223 // If so, evac_in_progress would be unset by collection set preparation code.
224 if (heap->is_evacuation_in_progress()) {
225 // Concurrently evacuate
226 entry_evacuate();
227 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
228 return false;
229 }
230
231 // Perform update-refs phase.
232 entry_concurrent_update_refs_prepare(heap);
233
234 if (ShenandoahHeap::heap()->mode()->is_generational()) {
235 entry_update_card_table();
236 }
237
238 if (ShenandoahVerify) {
239 vmop_entry_init_update_refs();
240 }
241
242 entry_update_refs();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 // Concurrent update thread roots
248 entry_update_thread_roots();
249 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
250 return false;
251 }
252
253 vmop_entry_final_update_refs();
254
255 // Update references freed up collection set, kick the cleanup to reclaim the space.
256 entry_cleanup_complete();
257 } else {
258 _abbreviated = true;
259
260 if (heap->mode()->is_generational()) {
261 entry_complete_abbreviated_cycle();
262
263 // If the promote-in-place operation was cancelled, we can have the degenerated
264 // cycle complete the operation. It will see that no evacuations are in progress,
265 // and that there are regions wanting promotion. The risk with not handling the
266 // cancellation would be failing to restore top for these regions and leaving
267 // them unable to serve allocations for the old generation.
268 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
269 return false;
270 }
271 }
272
273 // In normal cycle, final-update-refs would verify at the end of the cycle.
274 // In abbreviated cycle, we need to verify separately.
275 if (ShenandoahVerify) {
276 vmop_entry_final_verify();
277 }
278 }
279
280 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
281 // abbreviated cycle.
282 if (heap->mode()->is_generational()) {
283 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
284 }
285
286 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
287 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
288 // reducing the likelihood that GC will degenerate.
289 entry_reset_after_collect();
290
291 return true;
292 }
293
294 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
295 shenandoah_assert_generational();
296
297 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
298
299 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
300 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent complete abbreviated cycle", "");
301 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
302 EventMark em("%s", msg);
303
304 ShenandoahWorkerScope scope(heap->workers(),
305 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
306 msg);
307
308 heap->try_inject_pin();
309 // We chose not to evacuate because we found sufficient immediate garbage.
310 // However, there may still be regions to promote in place, so do that now.
311 if (heap->old_generation()->has_in_place_promotions()) {
312 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
313 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
314 heap->promote_regions_in_place(_generation, true);
315 }
316
317 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
318 // the control thread will detect it on its next iteration and run a degenerated young cycle.
319 if (!heap->cancelled_gc() && !_generation->is_old()) {
320 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
321 heap->update_region_ages(_generation->complete_marking_context());
322 }
323 }
324
325 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
326 ShenandoahHeap* const heap = ShenandoahHeap::heap();
327 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
328 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
329
330 heap->try_inject_alloc_failure();
331 VM_ShenandoahInitMark op(this);
332 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
333 }
334
335 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
336 ShenandoahHeap* const heap = ShenandoahHeap::heap();
337 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
338 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
339
340 heap->try_inject_alloc_failure();
341 VM_ShenandoahFinalMarkStartEvac op(this);
342 VMThread::execute(&op); // jump to entry_final_mark under safepoint
343 heap->try_inject_pin();
344 }
345
346 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
347 ShenandoahHeap* const heap = ShenandoahHeap::heap();
348 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
349 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
350
351 heap->try_inject_alloc_failure();
352 heap->try_inject_pin();
353 VM_ShenandoahInitUpdateRefs op(this);
354 VMThread::execute(&op);
355 }
356
357 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
358 ShenandoahHeap* const heap = ShenandoahHeap::heap();
359 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
360 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
361
362 heap->try_inject_alloc_failure();
363 heap->try_inject_pin();
364 VM_ShenandoahFinalUpdateRefs op(this);
365 VMThread::execute(&op);
366 }
367
368 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
369 ShenandoahHeap* const heap = ShenandoahHeap::heap();
370 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
371 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
372
373 // This phase does not use workers, no need for setup
374 heap->try_inject_pin();
375 VM_ShenandoahFinalRoots op(this);
376 VMThread::execute(&op);
377 }
378
379 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
380 ShenandoahHeap* const heap = ShenandoahHeap::heap();
381 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
382 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
383
384 // This phase does not use workers, no need for setup
385 heap->try_inject_alloc_failure();
386 heap->try_inject_pin();
387 VM_ShenandoahFinalVerify op(this);
388 VMThread::execute(&op);
389 }
390
391 void ShenandoahConcurrentGC::entry_init_mark() {
392 ShenandoahHeap* const heap = ShenandoahHeap::heap();
393 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
394
395 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Mark", "");
396 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
397 EventMark em("%s", msg);
398
399 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
400 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
401 "init marking");
402
403 op_init_mark();
404 }
405
406 void ShenandoahConcurrentGC::entry_final_mark() {
407 ShenandoahHeap* const heap = ShenandoahHeap::heap();
408 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
409 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
410
411 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Mark", "");
412 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
413 EventMark em("%s", msg);
414
415 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
416 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
417 "final marking");
418
419 op_final_mark();
420 }
421
422 void ShenandoahConcurrentGC::entry_init_update_refs() {
423 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Update Refs", "");
424 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
425 EventMark em("%s", msg);
426
427 // No workers used in this phase, no setup required
428 op_init_update_refs();
429 }
430
431 void ShenandoahConcurrentGC::entry_final_update_refs() {
432 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Update Refs", "");
433 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
434 EventMark em("%s", msg);
435
436 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
437 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
438 "final reference update");
439
440 op_final_update_refs();
441 }
442
443 void ShenandoahConcurrentGC::entry_final_verify() {
444 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Verify Final", "");
445 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
446 EventMark em("%s", msg);
447
448 op_verify_final();
449 }
450
451 void ShenandoahConcurrentGC::entry_reset() {
452 ShenandoahHeap* const heap = ShenandoahHeap::heap();
453 heap->release_injected_pins();
454 heap->try_inject_alloc_failure();
455
456 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
457 {
458 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset", "");
459 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
460 EventMark em("%s", msg);
461
462 ShenandoahWorkerScope scope(heap->workers(),
463 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
464 msg);
465 op_reset();
466 }
467 }
468
469 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
470 if (_generation->is_young()) {
471 ShenandoahHeap* const heap = ShenandoahHeap::heap();
472 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
473
474 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent remembered set scanning", "");
475 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
476 EventMark em("%s", msg);
477
478 ShenandoahWorkerScope scope(heap->workers(),
479 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
480 msg);
481
482 heap->try_inject_alloc_failure();
483 _generation->scan_remembered_set(true /* is_concurrent */);
484 }
485 }
486
487 void ShenandoahConcurrentGC::entry_mark_roots() {
488 ShenandoahHeap* const heap = ShenandoahHeap::heap();
489 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
490 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking roots", "");
491 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
492 EventMark em("%s", msg);
493
494 ShenandoahWorkerScope scope(heap->workers(),
495 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
496 "concurrent marking roots");
497
498 heap->try_inject_alloc_failure();
499 op_mark_roots();
500 }
501
502 void ShenandoahConcurrentGC::entry_mark() {
503 ShenandoahHeap* const heap = ShenandoahHeap::heap();
504 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
505 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
506
507 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
508 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking", "");
509 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
510 EventMark em("%s", msg);
511
512 ShenandoahWorkerScope scope(heap->workers(),
513 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
514 "concurrent marking");
515
516 heap->try_inject_alloc_failure();
517 op_mark();
518 heap->try_inject_pin();
519 }
520
521 void ShenandoahConcurrentGC::entry_thread_roots() {
522 ShenandoahHeap* const heap = ShenandoahHeap::heap();
523 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent thread roots", "");
524 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
525 EventMark em("%s", msg);
526
527 ShenandoahWorkerScope scope(heap->workers(),
528 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
529 msg);
530
531 heap->try_inject_alloc_failure();
532 heap->try_inject_pin();
533 op_thread_roots();
534 }
535
536 void ShenandoahConcurrentGC::entry_weak_refs() {
537 ShenandoahHeap* const heap = ShenandoahHeap::heap();
538 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak references", "");
539 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
540 EventMark em("%s", msg);
541
542 ShenandoahWorkerScope scope(heap->workers(),
543 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
544 "concurrent weak references");
545
546 heap->try_inject_alloc_failure();
547 heap->try_inject_pin();
548 op_weak_refs();
549 }
550
551 void ShenandoahConcurrentGC::entry_weak_roots() {
552 ShenandoahHeap* const heap = ShenandoahHeap::heap();
553 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
554 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak roots", "");
555 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
556 EventMark em("%s", msg);
557
558 ShenandoahWorkerScope scope(heap->workers(),
559 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
560 "concurrent weak root");
561
562 heap->try_inject_alloc_failure();
563 heap->try_inject_pin();
564 op_weak_roots();
565 }
566
567 void ShenandoahConcurrentGC::entry_class_unloading() {
568 ShenandoahHeap* const heap = ShenandoahHeap::heap();
569 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
570 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent class unloading", "");
571 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
572 EventMark em("%s", msg);
573
574 ShenandoahWorkerScope scope(heap->workers(),
575 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
576 "concurrent class unloading");
577
578 heap->try_inject_alloc_failure();
579 heap->try_inject_pin();
580 op_class_unloading();
581 }
582
583 void ShenandoahConcurrentGC::entry_strong_roots() {
584 ShenandoahHeap* const heap = ShenandoahHeap::heap();
585 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
586 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent strong roots", "");
587 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
588 EventMark em("%s", msg);
589
590 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
591
592 ShenandoahWorkerScope scope(heap->workers(),
593 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
594 "concurrent strong root");
595
596 heap->try_inject_alloc_failure();
597 heap->try_inject_pin();
598 op_strong_roots();
599 }
600
601 void ShenandoahConcurrentGC::entry_cleanup_early() {
602 ShenandoahHeap* const heap = ShenandoahHeap::heap();
603 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
604 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
605 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, false /* log_heap_usage */);
606 EventMark em("%s", msg);
607
608 // This phase does not use workers, no need for setup
609 heap->try_inject_alloc_failure();
610 heap->try_inject_pin();
611 op_cleanup_early();
612 if (!heap->is_evacuation_in_progress()) {
613 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
614 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
615 // during promote-in-place processing.
616 heap->rebuild_free_set(true /*concurrent*/);
617 }
618 }
619
620 void ShenandoahConcurrentGC::entry_evacuate() {
621 ShenandoahHeap* const heap = ShenandoahHeap::heap();
622 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
623 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent evacuation", "");
624 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
625 EventMark em("%s", msg);
626
627 ShenandoahWorkerScope scope(heap->workers(),
628 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
629 "concurrent evacuation");
630
631 heap->try_inject_alloc_failure();
632 heap->try_inject_pin();
633 op_evacuate();
634 }
635
636 void ShenandoahConcurrentGC::entry_update_thread_roots() {
637 ShenandoahHeap* const heap = ShenandoahHeap::heap();
638 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
639 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update thread roots", "");
640 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
641 EventMark em("%s", msg);
642
643 // No workers used in this phase, no setup required
644 heap->try_inject_alloc_failure();
645 heap->try_inject_pin();
646 op_update_thread_roots();
647 }
648
649 void ShenandoahConcurrentGC::entry_update_refs() {
650 ShenandoahHeap* const heap = ShenandoahHeap::heap();
651 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
652 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update references", "");
653 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
654 EventMark em("%s", msg);
655
656 ShenandoahWorkerScope scope(heap->workers(),
657 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
658 "concurrent reference update");
659
660 heap->try_inject_alloc_failure();
661 heap->try_inject_pin();
662 op_update_refs();
663 }
664
665 void ShenandoahConcurrentGC::entry_cleanup_complete() {
666 ShenandoahHeap* const heap = ShenandoahHeap::heap();
667 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
668 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
669 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, false /* log_heap_usage */);
670 EventMark em("%s", msg);
671
672 // This phase does not use workers, no need for setup
673 heap->try_inject_alloc_failure();
674 op_cleanup_complete();
675 }
676
677 void ShenandoahConcurrentGC::entry_reset_after_collect() {
678 ShenandoahHeap* const heap = ShenandoahHeap::heap();
679 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
680 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset after collect", "");
681 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
682 EventMark em("%s", msg);
683
684 op_reset_after_collect();
685 }
686
687 void ShenandoahConcurrentGC::op_reset() {
688 ShenandoahHeap* const heap = ShenandoahHeap::heap();
689
690 // If it is old GC bootstrap cycle, always clear bitmap for global gen
691 // to ensure bitmap for old gen is clear for old GC cycle after this.
692 if (_do_old_gc_bootstrap) {
693 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
694 heap->global_generation()->prepare_gc();
695 } else {
696 _generation->prepare_gc();
697 }
698
699 if (heap->mode()->is_generational()) {
700 heap->old_generation()->card_scan()->mark_read_table_as_clean();
701 }
702 }
703
704 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
705 private:
706 ShenandoahMarkingContext* const _ctx;
707 public:
708 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
709
710 void heap_region_do(ShenandoahHeapRegion* r) {
711 assert(!r->has_live(), "Region %zu should have no live data", r->index());
712 if (r->is_active()) {
713 // Check if region needs updating its TAMS. We have updated it already during concurrent
714 // reset, so it is very likely we don't need to do another write here. Since most regions
715 // are not "active", this path is relatively rare.
716 if (_ctx->top_at_mark_start(r) != r->top()) {
717 _ctx->capture_top_at_mark_start(r);
718 }
719 } else {
720 assert(_ctx->top_at_mark_start(r) == r->top(),
721 "Region %zu should already have correct TAMS", r->index());
722 }
723 }
724
725 bool is_thread_safe() { return true; }
726 };
727
728 void ShenandoahConcurrentGC::start_mark() {
729 _mark.start_mark();
730 }
731
732 void ShenandoahConcurrentGC::op_init_mark() {
733 ShenandoahHeap* const heap = ShenandoahHeap::heap();
734 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
735 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
736
737 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
738 assert(!_generation->is_mark_complete(), "should not be complete");
739 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
740
741 if (heap->mode()->is_generational()) {
742 if (_generation->is_global()) {
743 heap->old_generation()->cancel_gc();
744 }
745
746 {
747 // After we swap card table below, the write-table is all clean, and the read table holds
748 // cards dirty prior to the start of GC. Young and bootstrap collection will update
749 // the write card table as a side effect of remembered set scanning. Global collection will
750 // update the card table as a side effect of global marking of old objects.
751 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
752 _generation->swap_card_tables();
753 }
754 }
755
756 if (ShenandoahVerify) {
757 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
758 heap->verifier()->verify_before_concmark(_generation);
759 }
760
761 if (VerifyBeforeGC) {
762 Universe::verify();
763 }
764
765 _generation->set_concurrent_mark_in_progress(true);
766
767 start_mark();
768
769 if (_do_old_gc_bootstrap) {
770 shenandoah_assert_generational();
771 // Update region state for both young and old regions
772 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
773 ShenandoahInitMarkUpdateRegionStateClosure cl;
774 heap->parallel_heap_region_iterate(&cl);
775 heap->old_generation()->ref_processor()->reset_thread_locals();
776 } else {
777 // Update region state for only young regions
778 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
779 ShenandoahInitMarkUpdateRegionStateClosure cl;
780 _generation->parallel_heap_region_iterate(&cl);
781 }
782
783 // Weak reference processing
784 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
785 rp->reset_thread_locals();
786
787 // Make above changes visible to worker threads
788 OrderAccess::fence();
789
790 // Arm nmethods/stack for concurrent processing
791 CodeCache::arm_all_nmethods();
792
793 {
794 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
795 heap->propagate_gc_state_to_all_threads();
796 }
797 }
798
799 void ShenandoahConcurrentGC::op_mark_roots() {
800 _mark.mark_concurrent_roots();
801 }
802
803 void ShenandoahConcurrentGC::op_mark() {
804 _mark.concurrent_mark();
805 }
806
807 void ShenandoahConcurrentGC::op_final_mark() {
808 ShenandoahHeap* const heap = ShenandoahHeap::heap();
809 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
810 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
811
812 if (ShenandoahVerify) {
813 heap->verifier()->verify_roots_no_forwarded(_generation);
814 }
815
816 if (!heap->cancelled_gc()) {
817 _mark.finish_mark();
818 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
819
820 // Notify JVMTI that the tagmap table will need cleaning.
821 JvmtiTagMap::set_needs_cleaning();
822
823 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
824 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
825 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
826 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
827
828 // Has to be done after cset selection
829 heap->prepare_concurrent_roots();
830
831 if (!heap->collection_set()->is_empty()) {
832 LogTarget(Debug, gc, cset) lt;
833 if (lt.is_enabled()) {
834 ResourceMark rm;
835 LogStream ls(lt);
836 heap->collection_set()->print_on(&ls);
837 }
838
839 if (ShenandoahVerify) {
840 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
841 heap->verifier()->verify_before_evacuation(_generation);
842 }
843
844 heap->set_evacuation_in_progress(true);
845 // From here on, we need to update references.
846 heap->set_has_forwarded_objects(true);
847 } else {
848 if (ShenandoahVerify) {
849 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
850 if (has_in_place_promotions(heap)) {
851 heap->verifier()->verify_after_concmark_with_promotions(_generation);
852 } else {
853 heap->verifier()->verify_after_concmark(_generation);
854 }
855 }
856 }
857 }
858
859 // Arm nmethods/stack for concurrent processing
860 CodeCache::arm_all_nmethods();
861
862 {
863 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
864 heap->propagate_gc_state_to_all_threads();
865 }
866 }
867
868 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
869 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
870 }
871
872 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
873 private:
874 OopClosure* const _oops;
875 public:
876 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
877
878 void do_thread(Thread* thread) override {
879 JavaThread* const jt = JavaThread::cast(thread);
880 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
881 }
882 };
883
884 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
885 private:
886 ShenandoahJavaThreadsIterator _java_threads;
887
888 public:
889 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
890 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
891 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
892 }
893
894 void work(uint worker_id) override {
895 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
896 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
897 _java_threads.threads_do(&thr_cl, worker_id);
898 }
899 };
900
901 void ShenandoahConcurrentGC::op_thread_roots() {
902 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
903 assert(heap->is_evacuation_in_progress(), "Checked by caller");
904 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
905 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
906 heap->workers()->run_task(&task);
907 }
908
909 void ShenandoahConcurrentGC::op_weak_refs() {
910 ShenandoahHeap* const heap = ShenandoahHeap::heap();
911 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
912 // Concurrent weak refs processing
913 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
914 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
915 ShenandoahBreakpoint::at_after_reference_processing_started();
916 }
917 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
918 }
919
920 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
921 private:
922 ShenandoahHeap* const _heap;
923 ShenandoahGeneration* const _generation;
924 ShenandoahMarkingContext* const _mark_context;
925 bool _evac_in_progress;
926 Thread* const _thread;
927
928 public:
929 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
930 void do_oop(oop* p);
931 void do_oop(narrowOop* p);
932 };
933
934 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
935 _heap(ShenandoahHeap::heap()),
936 _generation(generation),
937 _mark_context(ShenandoahHeap::heap()->marking_context()),
938 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
939 _thread(Thread::current()) {
940 }
941
942 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
943 const oop obj = RawAccess<>::oop_load(p);
944 if (!CompressedOops::is_null(obj)) {
945 if (!_mark_context->is_marked(obj)) {
946 if (_generation->contains(obj)) {
947 // Note: The obj is dead here. Do not touch it, just clear.
948 ShenandoahHeap::atomic_clear_oop(p, obj);
949 }
950 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
951 oop resolved = ShenandoahForwarding::get_forwardee(obj);
952 if (resolved == obj) {
953 resolved = _heap->evacuate_object(obj, _thread);
954 }
955 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
956 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
957 }
958 }
959 }
960
961 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
962 ShouldNotReachHere();
963 }
964
965 class ShenandoahIsCLDAliveClosure : public CLDClosure {
966 public:
967 void do_cld(ClassLoaderData* cld) {
968 cld->is_alive();
969 }
970 };
971
972 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
973 public:
974 void do_nmethod(nmethod* n) {
975 n->is_unloading();
976 }
977 };
978
979 // This task not only evacuates/updates marked weak roots, but also "null"
980 // dead weak roots.
981 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
982 private:
983 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
984
985 // Roots related to concurrent class unloading
986 ShenandoahClassLoaderDataRoots<true /* concurrent */>
987 _cld_roots;
988 ShenandoahConcurrentNMethodIterator _nmethod_itr;
989 ShenandoahGeneration* _generation;
990 ShenandoahPhaseTimings::Phase _phase;
991
992 public:
993 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
994 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
995 _vm_roots(phase),
996 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
997 _nmethod_itr(ShenandoahCodeRoots::table()),
998 _generation(generation),
999 _phase(phase) {}
1000
1001 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1002 // Notify runtime data structures of potentially dead oops
1003 _vm_roots.report_num_dead();
1004 }
1005
1006 void work(uint worker_id) override {
1007 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1008 SuspendibleThreadSetJoiner sts_join;
1009 {
1010 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1011 // may race against OopStorage::release() calls.
1012 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
1013 _vm_roots.oops_do(&cl, worker_id);
1014 }
1015
1016 // If we are going to perform concurrent class unloading later on, we need to
1017 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
1018 // can clean up immediate garbage sooner.
1019 if (ShenandoahHeap::heap()->unload_classes()) {
1020 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
1021 // CLD's holder or evacuate it.
1022 {
1023 ShenandoahIsCLDAliveClosure is_cld_alive;
1024 _cld_roots.cld_do(&is_cld_alive, worker_id);
1025 }
1026
1027 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1028 // The closure calls nmethod->is_unloading(). The is_unloading
1029 // state is cached, therefore, during concurrent class unloading phase,
1030 // we will not touch the metadata of unloading nmethods
1031 {
1032 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1033 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1034 _nmethod_itr.nmethods_do(&is_nmethod_alive);
1035 }
1036 }
1037 }
1038 };
1039
1040 void ShenandoahConcurrentGC::op_weak_roots() {
1041 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1042 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1043 {
1044 // Concurrent weak root processing
1045 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1046 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1047 heap->workers()->run_task(&task);
1048 }
1049
1050 {
1051 // It is possible for mutators executing the load reference barrier to have
1052 // loaded an oop through a weak handle that has since been nulled out by
1053 // weak root processing. Handshaking here forces them to complete the
1054 // barrier before the GC cycle continues and does something that would
1055 // change the evaluation of the barrier (for example, resetting the TAMS
1056 // on trashed regions could make an oop appear to be marked _after_ the
1057 // region has been recycled).
1058 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1059 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1060 }
1061 }
1062
1063 void ShenandoahConcurrentGC::op_class_unloading() {
1064 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1065 assert (heap->is_concurrent_weak_root_in_progress() &&
1066 heap->unload_classes(),
1067 "Checked by caller");
1068 heap->do_class_unloading();
1069 }
1070
1071 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1072 private:
1073 ShenandoahEvacuateUpdateMetadataClosure _cl;
1074
1075 public:
1076 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1077
1078 void do_nmethod(nmethod* n) {
1079 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1080 ShenandoahNMethodLocker locker(data->lock());
1081 data->oops_do(&_cl, /* fix_relocations = */ true);
1082 }
1083 };
1084
1085 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1086 private:
1087 ShenandoahPhaseTimings::Phase _phase;
1088 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1089 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1090 _cld_roots;
1091 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1092
1093 public:
1094 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1095 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1096 _phase(phase),
1097 _vm_roots(phase),
1098 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1099 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1100
1101 void work(uint worker_id) {
1102 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1103 {
1104 {
1105 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1106 // may race against OopStorage::release() calls.
1107 ShenandoahContextEvacuateUpdateRootsClosure cl;
1108 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1109 }
1110
1111 {
1112 ShenandoahEvacuateUpdateMetadataClosure cl;
1113 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1114 _cld_roots.cld_do(&clds, worker_id);
1115 }
1116 }
1117
1118 if (!ShenandoahHeap::heap()->unload_classes()) {
1119 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1120 ShenandoahEvacUpdateCodeCacheClosure cl;
1121 _nmethod_itr.nmethods_do(&cl);
1122 }
1123 }
1124 };
1125
1126 void ShenandoahConcurrentGC::op_strong_roots() {
1127 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1128 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1129 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1130 heap->workers()->run_task(&task);
1131 heap->set_concurrent_strong_root_in_progress(false);
1132 }
1133
1134 void ShenandoahConcurrentGC::op_cleanup_early() {
1135 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1136 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1137 "cleanup early.");
1138 ShenandoahHeap::heap()->recycle_trash();
1139 }
1140
1141 void ShenandoahConcurrentGC::op_evacuate() {
1142 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1143 }
1144
1145 void ShenandoahConcurrentGC::op_init_update_refs() {
1146 if (ShenandoahVerify) {
1147 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1148 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1149 heap->verifier()->verify_before_update_refs(_generation);
1150 }
1151 }
1152
1153 void ShenandoahConcurrentGC::op_update_refs() {
1154 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1155 }
1156
1157 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1158 private:
1159 // This closure runs when thread is stopped for handshake, which means
1160 // we can use non-concurrent closure here, as long as it only updates
1161 // locations modified by the thread itself, i.e. stack locations.
1162 ShenandoahNonConcUpdateRefsClosure _cl;
1163 public:
1164 ShenandoahUpdateThreadHandshakeClosure();
1165 void do_thread(Thread* thread) override;
1166 };
1167
1168 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1169 HandshakeClosure("Shenandoah Update Thread Roots") {
1170 }
1171
1172 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1173 if (thread->is_Java_thread()) {
1174 JavaThread* jt = JavaThread::cast(thread);
1175 ResourceMark rm;
1176 jt->oops_do(&_cl, nullptr);
1177 }
1178 }
1179
1180 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1181 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1182 // to pick up overwritten pointers that point into a cset region. If these pointers
1183 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1184 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1185 //
1186 // Therefore, at the end of update refs, we use this closure to update the thread roots
1187 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1188 // anything that has already been marked or anything that points to a region which is
1189 // not old. We do not need to worry about ABA situations where a region may become old
1190 // after the pointer is enqueued but before it is filtered. There are only two ways a
1191 // region may become old:
1192 // 1. The region is promoted in place. This is safe because such regions will never
1193 // be in the collection set. If this happens, the pointer will be preserved, essentially
1194 // becoming part of the old snapshot.
1195 // 2. The region is allocated during evacuation of old. This is also not a concern because
1196 // we haven't yet finished marking old so no mixed evacuations will happen.
1197 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1198 ShenandoahFlushSATB _flush_all_satb;
1199
1200 public:
1201 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1202 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1203 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1204 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1205 "Should be filtering pointers outside of old during old marking");
1206 }
1207
1208 void do_thread(Thread* thread) override {
1209 _update_roots.do_thread(thread);
1210 _flush_all_satb.do_thread(thread);
1211 }
1212 };
1213
1214 void ShenandoahConcurrentGC::op_update_thread_roots() {
1215 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1216 if (heap->is_concurrent_old_mark_in_progress()) {
1217 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1218 Handshake::execute(&cl);
1219 } else {
1220 ShenandoahUpdateThreadHandshakeClosure cl;
1221 Handshake::execute(&cl);
1222 }
1223 }
1224
1225 void ShenandoahConcurrentGC::op_final_update_refs() {
1226 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1227 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1228 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1229
1230 heap->finish_concurrent_roots();
1231
1232 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1233 // everything.
1234 if (heap->cancelled_gc()) {
1235 heap->clear_cancelled_gc();
1236 }
1237
1238 // Has to be done before cset is clear
1239 if (ShenandoahVerify) {
1240 heap->verifier()->verify_roots_in_to_space(_generation);
1241 }
1242
1243 // If we are running in generational mode, this will also age active regions that
1244 // haven't been used for allocation.
1245 heap->update_heap_region_states(true /*concurrent*/);
1246
1247 heap->set_update_refs_in_progress(false);
1248 heap->set_has_forwarded_objects(false);
1249
1250 if (ShenandoahVerify) {
1251 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1252 heap->verifier()->verify_after_update_refs(_generation);
1253 }
1254
1255 if (VerifyAfterGC) {
1256 Universe::verify();
1257 }
1258
1259 heap->rebuild_free_set(true /*concurrent*/);
1260 _generation->heuristics()->start_idle_span();
1261
1262 // Final pause: update GC barriers to idle state.
1263 CodeCache::arm_all_nmethods();
1264
1265 {
1266 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1267 heap->propagate_gc_state_to_all_threads();
1268 }
1269 }
1270
1271 void ShenandoahConcurrentGC::entry_final_roots() {
1272 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1273 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Roots", "");
1274 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1275 EventMark em("%s", msg);
1276
1277 heap->op_final_roots();
1278 }
1279
1280 void ShenandoahConcurrentGC::op_verify_final() {
1281 assert(ShenandoahVerify, "Should have been checked before");
1282 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1283 heap->verifier()->verify_after_gc(_generation);
1284 }
1285
1286 void ShenandoahConcurrentGC::op_cleanup_complete() {
1287 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1288 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1289 "cleanup complete.");
1290 ShenandoahHeap::heap()->recycle_trash();
1291 }
1292
1293 void ShenandoahConcurrentGC::op_reset_after_collect() {
1294 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1295 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1296 "reset after collection.");
1297
1298 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1299 if (heap->mode()->is_generational()) {
1300 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1301 // the young generation intact. In particular, reference processing in the old generation may potentially
1302 // need the reachability of a young generation referent of a Reference object in the old generation.
1303 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1304 heap->young_generation()->reset_mark_bitmap<false>();
1305 }
1306 } else {
1307 _generation->reset_mark_bitmap<false>();
1308 }
1309 }
1310
1311 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1312 if (ShenandoahHeap::heap()->cancelled_gc()) {
1313 _degen_point = point;
1314 return true;
1315 }
1316 return false;
1317 }