1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
37 #include "gc/shenandoah/shenandoahGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahLock.hpp"
40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
42 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
52 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
53 #include "memory/allocation.hpp"
54 #include "prims/jvmtiTagMap.hpp"
55 #include "runtime/icache.hpp"
56 #include "runtime/vmThread.hpp"
57 #include "utilities/events.hpp"
58
59 // Breakpoint support
60 class ShenandoahBreakpointGCScope : public StackObj {
61 private:
62 const GCCause::Cause _cause;
63 public:
64 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
65 if (cause == GCCause::_wb_breakpoint) {
66 ShenandoahBreakpoint::start_gc();
67 ShenandoahBreakpoint::at_before_gc();
68 }
69 }
70
71 ~ShenandoahBreakpointGCScope() {
72 if (_cause == GCCause::_wb_breakpoint) {
73 ShenandoahBreakpoint::at_after_gc();
74 }
75 }
76 };
77
78 class ShenandoahBreakpointMarkScope : public StackObj {
79 private:
80 const GCCause::Cause _cause;
81 public:
82 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
83 if (_cause == GCCause::_wb_breakpoint) {
84 ShenandoahBreakpoint::at_after_marking_started();
85 }
86 }
87
88 ~ShenandoahBreakpointMarkScope() {
89 if (_cause == GCCause::_wb_breakpoint) {
90 ShenandoahBreakpoint::at_before_marking_completed();
91 }
92 }
93 };
94
95 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
96 ShenandoahGC(generation),
97 _mark(generation),
98 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
99 _abbreviated(false),
100 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
101 }
102
103 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
104 return _degen_point;
105 }
106
107 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
108 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
109 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent init update refs", "");
110 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
111 EventMark em("%s", msg);
112
113 heap->try_inject_pin();
114 // Evacuation is complete, retire gc labs and change gc state
115 heap->concurrent_prepare_for_update_refs();
116 }
117
118 void ShenandoahConcurrentGC::entry_update_card_table() {
119 ShenandoahHeap* const heap = ShenandoahHeap::heap();
120 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
121 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update cards", "");
122 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
123 EventMark em("%s", msg);
124
125 ShenandoahWorkerScope scope(heap->workers(),
126 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
127 "concurrent update cards");
128
129 heap->try_inject_pin();
130 // Heap needs to be parsable here.
131 // Also, parallel heap region iterate must have a phase set.
132 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
133 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
134 }
135
136 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
137 ShenandoahHeap* const heap = ShenandoahHeap::heap();
138 _generation->ref_processor()->set_soft_reference_policy(
139 GCCause::should_clear_all_soft_refs(cause));
140
141 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent GC", "");
142 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_gc, /* log_heap_usage = */ true);
143
144 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
145
146 // Reset for upcoming marking
147 entry_reset();
148
149 // Start initial mark under STW
150 vmop_entry_init_mark();
151
152 {
153 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
154
155 // Reset task queue stats here, rather than in mark_concurrent_roots,
156 // because remembered set scan will `push` oops into the queues and
157 // resetting after this happens will lose those counts.
158 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
159
160 // Concurrent remembered set scanning
161 entry_scan_remembered_set();
162
163 // Concurrent mark roots
164 entry_mark_roots();
165 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
166 return false;
167 }
168
169 // Continue concurrent mark
170 entry_mark();
171 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
172 return false;
173 }
174 }
175
176 // Complete marking under STW, and start evacuation
177 vmop_entry_final_mark();
178
179 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
180 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
181 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
182 // from that phase.
183 if (_generation->is_concurrent_mark_in_progress()) {
184 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
185 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
186 return false;
187 }
188
189 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
190
191 // Finish all thread/stack roots if needed. This completes stack watermark processing.
192 if (heap->is_evacuation_in_progress()) {
193 entry_thread_roots();
194 }
195
196 // Process weak roots that might still point to regions that would be broken by cleanup.
197 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
198 entry_weak_refs();
199 entry_weak_roots();
200
201 // Perform concurrent class unloading before any regions get recycled. Class unloading may
202 // need to inspect unmarked objects in trashed regions.
203 if (heap->unload_classes()) {
204 entry_class_unloading();
205 }
206
207 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
208 // the space. This would be the last action if there is nothing to evacuate. Note that
209 // we will not age young-gen objects in the case that we skip evacuation.
210 entry_cleanup_early();
211
212 // Processing strong roots
213 // This may be skipped if there is nothing to update/evacuate.
214 // If so, strong_root_in_progress would be unset.
215 if (heap->is_concurrent_strong_root_in_progress()) {
216 entry_strong_roots();
217 }
218
219 // Roots processing is complete, put the weak roots flag down.
220 vmop_entry_final_roots();
221
222 // Continue the cycle with evacuation and optional update-refs.
223 // This may be skipped if there is nothing to evacuate.
224 // If so, evac_in_progress would be unset by collection set preparation code.
225 if (heap->is_evacuation_in_progress()) {
226 // Concurrently evacuate
227 entry_evacuate();
228 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
229 return false;
230 }
231
232 // Perform update-refs phase.
233 entry_concurrent_update_refs_prepare(heap);
234
235 if (ShenandoahHeap::heap()->mode()->is_generational()) {
236 entry_update_card_table();
237 }
238
239 if (ShenandoahVerify) {
240 vmop_entry_init_update_refs();
241 }
242
243 entry_update_refs();
244 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
245 return false;
246 }
247
248 // Concurrent update thread roots
249 entry_update_thread_roots();
250 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
251 return false;
252 }
253
254 vmop_entry_final_update_refs();
255
256 // Update references freed up collection set, kick the cleanup to reclaim the space.
257 entry_cleanup_complete();
258 } else {
259 _abbreviated = true;
260
261 if (heap->mode()->is_generational()) {
262 entry_complete_abbreviated_cycle();
263
264 // If the promote-in-place operation was cancelled, we can have the degenerated
265 // cycle complete the operation. It will see that no evacuations are in progress,
266 // and that there are regions wanting promotion. The risk with not handling the
267 // cancellation would be failing to restore top for these regions and leaving
268 // them unable to serve allocations for the old generation.
269 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
270 return false;
271 }
272 }
273
274 // In normal cycle, final-update-refs would verify at the end of the cycle.
275 // In abbreviated cycle, we need to verify separately.
276 if (ShenandoahVerify) {
277 vmop_entry_final_verify();
278 }
279 }
280
281 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
282 // abbreviated cycle.
283 if (heap->mode()->is_generational()) {
284 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
285 }
286
287 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
288 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
289 // reducing the likelihood that GC will degenerate.
290 entry_reset_after_collect();
291
292 return true;
293 }
294
295 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
296 shenandoah_assert_generational();
297
298 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
299
300 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
301 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent complete abbreviated cycle", "");
302 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
303 EventMark em("%s", msg);
304
305 ShenandoahWorkerScope scope(heap->workers(),
306 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
307 msg);
308
309 heap->try_inject_pin();
310 // We chose not to evacuate because we found sufficient immediate garbage.
311 // However, there may still be regions to promote in place, so do that now.
312 if (heap->old_generation()->has_in_place_promotions()) {
313 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
314 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
315 heap->promote_regions_in_place(_generation, true);
316 }
317
318 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
319 // the control thread will detect it on its next iteration and run a degenerated young cycle.
320 if (!heap->cancelled_gc() && !_generation->is_old()) {
321 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
322 heap->update_region_ages(_generation->complete_marking_context());
323 }
324 }
325
326 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
327 ShenandoahHeap* const heap = ShenandoahHeap::heap();
328 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
329 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
330
331 heap->try_inject_alloc_failure();
332 VM_ShenandoahInitMark op(this);
333 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
334 }
335
336 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
337 ShenandoahHeap* const heap = ShenandoahHeap::heap();
338 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
339 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
340
341 heap->try_inject_alloc_failure();
342 VM_ShenandoahFinalMarkStartEvac op(this);
343 VMThread::execute(&op); // jump to entry_final_mark under safepoint
344 heap->try_inject_pin();
345 }
346
347 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
348 ShenandoahHeap* const heap = ShenandoahHeap::heap();
349 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
350 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
351
352 heap->try_inject_alloc_failure();
353 heap->try_inject_pin();
354 VM_ShenandoahInitUpdateRefs op(this);
355 VMThread::execute(&op);
356 }
357
358 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
359 ShenandoahHeap* const heap = ShenandoahHeap::heap();
360 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
361 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
362
363 heap->try_inject_alloc_failure();
364 heap->try_inject_pin();
365 VM_ShenandoahFinalUpdateRefs op(this);
366 VMThread::execute(&op);
367 }
368
369 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
370 ShenandoahHeap* const heap = ShenandoahHeap::heap();
371 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
372 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
373
374 // This phase does not use workers, no need for setup
375 heap->try_inject_pin();
376 VM_ShenandoahFinalRoots op(this);
377 VMThread::execute(&op);
378 }
379
380 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
381 ShenandoahHeap* const heap = ShenandoahHeap::heap();
382 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
383 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
384
385 // This phase does not use workers, no need for setup
386 heap->try_inject_alloc_failure();
387 heap->try_inject_pin();
388 VM_ShenandoahFinalVerify op(this);
389 VMThread::execute(&op);
390 }
391
392 void ShenandoahConcurrentGC::entry_init_mark() {
393 ShenandoahHeap* const heap = ShenandoahHeap::heap();
394 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
395
396 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Mark", "");
397 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
398 EventMark em("%s", msg);
399
400 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
401 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
402 "init marking");
403
404 op_init_mark();
405 }
406
407 void ShenandoahConcurrentGC::entry_final_mark() {
408 ShenandoahHeap* const heap = ShenandoahHeap::heap();
409 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
410 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
411
412 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Mark", "");
413 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
414 EventMark em("%s", msg);
415
416 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
417 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
418 "final marking");
419
420 op_final_mark();
421 }
422
423 void ShenandoahConcurrentGC::entry_init_update_refs() {
424 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Update Refs", "");
425 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
426 EventMark em("%s", msg);
427
428 // No workers used in this phase, no setup required
429 op_init_update_refs();
430 }
431
432 void ShenandoahConcurrentGC::entry_final_update_refs() {
433 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Update Refs", "");
434 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
435 EventMark em("%s", msg);
436
437 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
438 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
439 "final reference update");
440
441 op_final_update_refs();
442 }
443
444 void ShenandoahConcurrentGC::entry_final_verify() {
445 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Verify Final", "");
446 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
447 EventMark em("%s", msg);
448
449 op_verify_final();
450 }
451
452 void ShenandoahConcurrentGC::entry_reset() {
453 ShenandoahHeap* const heap = ShenandoahHeap::heap();
454 heap->release_injected_pins();
455 heap->try_inject_alloc_failure();
456
457 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
458 {
459 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset", "");
460 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
461 EventMark em("%s", msg);
462
463 ShenandoahWorkerScope scope(heap->workers(),
464 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
465 msg);
466 op_reset();
467 }
468 }
469
470 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
471 if (_generation->is_young()) {
472 ShenandoahHeap* const heap = ShenandoahHeap::heap();
473 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
474
475 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent remembered set scanning", "");
476 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
477 EventMark em("%s", msg);
478
479 ShenandoahWorkerScope scope(heap->workers(),
480 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
481 msg);
482
483 heap->try_inject_alloc_failure();
484 _generation->scan_remembered_set(true /* is_concurrent */);
485 }
486 }
487
488 void ShenandoahConcurrentGC::entry_mark_roots() {
489 ShenandoahHeap* const heap = ShenandoahHeap::heap();
490 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
491 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking roots", "");
492 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
493 EventMark em("%s", msg);
494
495 ShenandoahWorkerScope scope(heap->workers(),
496 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
497 "concurrent marking roots");
498
499 heap->try_inject_alloc_failure();
500 op_mark_roots();
501 }
502
503 void ShenandoahConcurrentGC::entry_mark() {
504 ShenandoahHeap* const heap = ShenandoahHeap::heap();
505 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
506 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
507
508 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
509 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking", "");
510 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
511 EventMark em("%s", msg);
512
513 ShenandoahWorkerScope scope(heap->workers(),
514 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
515 "concurrent marking");
516
517 heap->try_inject_alloc_failure();
518 op_mark();
519 heap->try_inject_pin();
520 }
521
522 void ShenandoahConcurrentGC::entry_thread_roots() {
523 ShenandoahHeap* const heap = ShenandoahHeap::heap();
524 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent thread roots", "");
525 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
526 EventMark em("%s", msg);
527
528 ShenandoahWorkerScope scope(heap->workers(),
529 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
530 msg);
531
532 heap->try_inject_alloc_failure();
533 heap->try_inject_pin();
534 op_thread_roots();
535 }
536
537 void ShenandoahConcurrentGC::entry_weak_refs() {
538 ShenandoahHeap* const heap = ShenandoahHeap::heap();
539 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak references", "");
540 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
541 EventMark em("%s", msg);
542
543 ShenandoahWorkerScope scope(heap->workers(),
544 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
545 "concurrent weak references");
546
547 heap->try_inject_alloc_failure();
548 heap->try_inject_pin();
549 op_weak_refs();
550 }
551
552 void ShenandoahConcurrentGC::entry_weak_roots() {
553 ShenandoahHeap* const heap = ShenandoahHeap::heap();
554 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
555 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak roots", "");
556 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
557 EventMark em("%s", msg);
558
559 ShenandoahWorkerScope scope(heap->workers(),
560 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
561 "concurrent weak root");
562
563 heap->try_inject_alloc_failure();
564 heap->try_inject_pin();
565 op_weak_roots();
566 }
567
568 void ShenandoahConcurrentGC::entry_class_unloading() {
569 ShenandoahHeap* const heap = ShenandoahHeap::heap();
570 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
571 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent class unloading", "");
572 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
573 EventMark em("%s", msg);
574
575 ShenandoahWorkerScope scope(heap->workers(),
576 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
577 "concurrent class unloading");
578
579 heap->try_inject_alloc_failure();
580 heap->try_inject_pin();
581 op_class_unloading();
582 }
583
584 void ShenandoahConcurrentGC::entry_strong_roots() {
585 ShenandoahHeap* const heap = ShenandoahHeap::heap();
586 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
587 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent strong roots", "");
588 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
589 EventMark em("%s", msg);
590
591 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
592
593 ShenandoahWorkerScope scope(heap->workers(),
594 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
595 "concurrent strong root");
596
597 heap->try_inject_alloc_failure();
598 heap->try_inject_pin();
599 op_strong_roots();
600 }
601
602 void ShenandoahConcurrentGC::entry_cleanup_early() {
603 ShenandoahHeap* const heap = ShenandoahHeap::heap();
604 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
605 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
606 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, false /* log_heap_usage */);
607 EventMark em("%s", msg);
608
609 // This phase does not use workers, no need for setup
610 heap->try_inject_alloc_failure();
611 heap->try_inject_pin();
612 op_cleanup_early();
613 if (!heap->is_evacuation_in_progress()) {
614 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
615 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
616 // during promote-in-place processing.
617 heap->rebuild_free_set(true /*concurrent*/);
618 }
619 }
620
621 void ShenandoahConcurrentGC::entry_evacuate() {
622 ShenandoahHeap* const heap = ShenandoahHeap::heap();
623 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
624 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent evacuation", "");
625 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
626 EventMark em("%s", msg);
627
628 ShenandoahWorkerScope scope(heap->workers(),
629 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
630 "concurrent evacuation");
631
632 heap->try_inject_alloc_failure();
633 heap->try_inject_pin();
634 op_evacuate();
635 }
636
637 void ShenandoahConcurrentGC::entry_update_thread_roots() {
638 ShenandoahHeap* const heap = ShenandoahHeap::heap();
639 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
640 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update thread roots", "");
641 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
642 EventMark em("%s", msg);
643
644 // No workers used in this phase, no setup required
645 heap->try_inject_alloc_failure();
646 heap->try_inject_pin();
647 op_update_thread_roots();
648 }
649
650 void ShenandoahConcurrentGC::entry_update_refs() {
651 ShenandoahHeap* const heap = ShenandoahHeap::heap();
652 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
653 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update references", "");
654 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
655 EventMark em("%s", msg);
656
657 ShenandoahWorkerScope scope(heap->workers(),
658 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
659 "concurrent reference update");
660
661 heap->try_inject_alloc_failure();
662 heap->try_inject_pin();
663 op_update_refs();
664 }
665
666 void ShenandoahConcurrentGC::entry_cleanup_complete() {
667 ShenandoahHeap* const heap = ShenandoahHeap::heap();
668 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
669 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
670 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, false /* log_heap_usage */);
671 EventMark em("%s", msg);
672
673 // This phase does not use workers, no need for setup
674 heap->try_inject_alloc_failure();
675 op_cleanup_complete();
676 }
677
678 void ShenandoahConcurrentGC::entry_reset_after_collect() {
679 ShenandoahHeap* const heap = ShenandoahHeap::heap();
680 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
681 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset after collect", "");
682 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
683 EventMark em("%s", msg);
684
685 op_reset_after_collect();
686 }
687
688 void ShenandoahConcurrentGC::op_reset() {
689 ShenandoahHeap* const heap = ShenandoahHeap::heap();
690
691 // If it is old GC bootstrap cycle, always clear bitmap for global gen
692 // to ensure bitmap for old gen is clear for old GC cycle after this.
693 if (_do_old_gc_bootstrap) {
694 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
695 heap->global_generation()->prepare_gc();
696 } else {
697 _generation->prepare_gc();
698 }
699
700 if (heap->mode()->is_generational()) {
701 heap->old_generation()->card_scan()->mark_read_table_as_clean();
702 }
703 }
704
705 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
706 private:
707 ShenandoahMarkingContext* const _ctx;
708 public:
709 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
710
711 void heap_region_do(ShenandoahHeapRegion* r) {
712 assert(!r->has_live(), "Region %zu should have no live data", r->index());
713 if (r->is_active()) {
714 // Check if region needs updating its TAMS. We have updated it already during concurrent
715 // reset, so it is very likely we don't need to do another write here. Since most regions
716 // are not "active", this path is relatively rare.
717 if (_ctx->top_at_mark_start(r) != r->top()) {
718 _ctx->capture_top_at_mark_start(r);
719 }
720 } else {
721 assert(_ctx->top_at_mark_start(r) == r->top(),
722 "Region %zu should already have correct TAMS", r->index());
723 }
724 }
725
726 bool is_thread_safe() { return true; }
727 };
728
729 void ShenandoahConcurrentGC::start_mark() {
730 _mark.start_mark();
731 }
732
733 void ShenandoahConcurrentGC::op_init_mark() {
734 ShenandoahHeap* const heap = ShenandoahHeap::heap();
735 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
736 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
737
738 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
739 assert(!_generation->is_mark_complete(), "should not be complete");
740 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
741
742 if (heap->mode()->is_generational()) {
743 if (_generation->is_global()) {
744 heap->old_generation()->cancel_gc();
745 }
746
747 {
748 // After we swap card table below, the write-table is all clean, and the read table holds
749 // cards dirty prior to the start of GC. Young and bootstrap collection will update
750 // the write card table as a side effect of remembered set scanning. Global collection will
751 // update the card table as a side effect of global marking of old objects.
752 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
753 _generation->swap_card_tables();
754 }
755 }
756
757 if (ShenandoahVerify) {
758 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
759 heap->verifier()->verify_before_concmark(_generation);
760 }
761
762 if (VerifyBeforeGC) {
763 Universe::verify();
764 }
765
766 _generation->set_concurrent_mark_in_progress(true);
767
768 start_mark();
769
770 if (_do_old_gc_bootstrap) {
771 shenandoah_assert_generational();
772 // Update region state for both young and old regions
773 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
774 ShenandoahInitMarkUpdateRegionStateClosure cl;
775 heap->parallel_heap_region_iterate(&cl);
776 heap->old_generation()->ref_processor()->reset_thread_locals();
777 } else {
778 // Update region state for only young regions
779 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
780 ShenandoahInitMarkUpdateRegionStateClosure cl;
781 _generation->parallel_heap_region_iterate(&cl);
782 }
783
784 // Weak reference processing
785 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
786 rp->reset_thread_locals();
787
788 // Make above changes visible to worker threads
789 OrderAccess::fence();
790
791 // Arm nmethods/stack for concurrent processing
792 CodeCache::arm_all_nmethods();
793
794 {
795 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
796 heap->propagate_gc_state_to_all_threads();
797 }
798 }
799
800 void ShenandoahConcurrentGC::op_mark_roots() {
801 _mark.mark_concurrent_roots();
802 }
803
804 void ShenandoahConcurrentGC::op_mark() {
805 _mark.concurrent_mark();
806 }
807
808 void ShenandoahConcurrentGC::op_final_mark() {
809 ShenandoahHeap* const heap = ShenandoahHeap::heap();
810 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
811 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
812
813 if (ShenandoahVerify) {
814 heap->verifier()->verify_roots_no_forwarded(_generation);
815 }
816
817 if (!heap->cancelled_gc()) {
818 _mark.finish_mark();
819 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
820
821 // Notify JVMTI that the tagmap table will need cleaning.
822 JvmtiTagMap::set_needs_cleaning();
823
824 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
825 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
826 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
827 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
828
829 // Has to be done after cset selection
830 heap->prepare_concurrent_roots();
831
832 if (!heap->collection_set()->is_empty()) {
833 LogTarget(Debug, gc, cset) lt;
834 if (lt.is_enabled()) {
835 ResourceMark rm;
836 LogStream ls(lt);
837 heap->collection_set()->print_on(&ls);
838 }
839
840 if (ShenandoahVerify) {
841 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
842 heap->verifier()->verify_before_evacuation(_generation);
843 }
844
845 heap->set_evacuation_in_progress(true);
846 // From here on, we need to update references.
847 heap->set_has_forwarded_objects(true);
848 } else {
849 if (ShenandoahVerify) {
850 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
851 if (has_in_place_promotions(heap)) {
852 heap->verifier()->verify_after_concmark_with_promotions(_generation);
853 } else {
854 heap->verifier()->verify_after_concmark(_generation);
855 }
856 }
857 }
858 }
859
860 // Arm nmethods/stack for concurrent processing
861 CodeCache::arm_all_nmethods();
862
863 {
864 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
865 heap->propagate_gc_state_to_all_threads();
866 }
867 }
868
869 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
870 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
871 }
872
873 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
874 private:
875 OopClosure* const _oops;
876 public:
877 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
878
879 void do_thread(Thread* thread) override {
880 JavaThread* const jt = JavaThread::cast(thread);
881 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
882 }
883 };
884
885 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
886 private:
887 ShenandoahJavaThreadsIterator _java_threads;
888
889 public:
890 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
891 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
892 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
893 }
894
895 void work(uint worker_id) override {
896 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
897 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
898 _java_threads.threads_do(&thr_cl, worker_id);
899 }
900 };
901
902 void ShenandoahConcurrentGC::op_thread_roots() {
903 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
904 assert(heap->is_evacuation_in_progress(), "Checked by caller");
905 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
906 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
907 heap->workers()->run_task(&task);
908 }
909
910 void ShenandoahConcurrentGC::op_weak_refs() {
911 ShenandoahHeap* const heap = ShenandoahHeap::heap();
912 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
913 // Concurrent weak refs processing
914 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
915 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
916 ShenandoahBreakpoint::at_after_reference_processing_started();
917 }
918 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
919 }
920
921 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
922 private:
923 ShenandoahHeap* const _heap;
924 ShenandoahGeneration* const _generation;
925 ShenandoahMarkingContext* const _mark_context;
926 bool _evac_in_progress;
927 Thread* const _thread;
928
929 public:
930 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
931 void do_oop(oop* p);
932 void do_oop(narrowOop* p);
933 };
934
935 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
936 _heap(ShenandoahHeap::heap()),
937 _generation(generation),
938 _mark_context(ShenandoahHeap::heap()->marking_context()),
939 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
940 _thread(Thread::current()) {
941 }
942
943 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
944 const oop obj = RawAccess<>::oop_load(p);
945 if (!CompressedOops::is_null(obj)) {
946 if (!_mark_context->is_marked(obj)) {
947 if (_generation->contains(obj)) {
948 // Note: The obj is dead here. Do not touch it, just clear.
949 ShenandoahHeap::atomic_clear_oop(p, obj);
950 }
951 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
952 oop resolved = ShenandoahForwarding::get_forwardee(obj);
953 if (resolved == obj) {
954 resolved = _heap->evacuate_object(obj, _thread);
955 }
956 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
957 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
958 }
959 }
960 }
961
962 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
963 ShouldNotReachHere();
964 }
965
966 class ShenandoahIsCLDAliveClosure : public CLDClosure {
967 public:
968 void do_cld(ClassLoaderData* cld) {
969 cld->is_alive();
970 }
971 };
972
973 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
974 public:
975 void do_nmethod(nmethod* n) {
976 n->is_unloading();
977 }
978 };
979
980 // This task not only evacuates/updates marked weak roots, but also "null"
981 // dead weak roots.
982 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
983 private:
984 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
985
986 // Roots related to concurrent class unloading
987 ShenandoahClassLoaderDataRoots<true /* concurrent */>
988 _cld_roots;
989 ShenandoahConcurrentNMethodIterator _nmethod_itr;
990 ShenandoahGeneration* _generation;
991 ShenandoahPhaseTimings::Phase _phase;
992
993 public:
994 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
995 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
996 _vm_roots(phase),
997 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
998 _nmethod_itr(ShenandoahCodeRoots::table()),
999 _generation(generation),
1000 _phase(phase) {}
1001
1002 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1003 // Notify runtime data structures of potentially dead oops
1004 _vm_roots.report_num_dead();
1005 }
1006
1007 void work(uint worker_id) override {
1008 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1009 SuspendibleThreadSetJoiner sts_join;
1010 {
1011 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1012 // may race against OopStorage::release() calls.
1013 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
1014 _vm_roots.oops_do(&cl, worker_id);
1015 }
1016
1017 // If we are going to perform concurrent class unloading later on, we need to
1018 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
1019 // can clean up immediate garbage sooner.
1020 if (ShenandoahHeap::heap()->unload_classes()) {
1021 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
1022 // CLD's holder or evacuate it.
1023 {
1024 ShenandoahIsCLDAliveClosure is_cld_alive;
1025 _cld_roots.cld_do(&is_cld_alive, worker_id);
1026 }
1027
1028 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1029 // The closure calls nmethod->is_unloading(). The is_unloading
1030 // state is cached, therefore, during concurrent class unloading phase,
1031 // we will not touch the metadata of unloading nmethods
1032 {
1033 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1034 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1035 _nmethod_itr.nmethods_do(&is_nmethod_alive);
1036 }
1037 }
1038 }
1039 };
1040
1041 void ShenandoahConcurrentGC::op_weak_roots() {
1042 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1043 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1044 {
1045 // Concurrent weak root processing
1046 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1047 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1048 heap->workers()->run_task(&task);
1049 }
1050
1051 {
1052 // It is possible for mutators executing the load reference barrier to have
1053 // loaded an oop through a weak handle that has since been nulled out by
1054 // weak root processing. Handshaking here forces them to complete the
1055 // barrier before the GC cycle continues and does something that would
1056 // change the evaluation of the barrier (for example, resetting the TAMS
1057 // on trashed regions could make an oop appear to be marked _after_ the
1058 // region has been recycled).
1059 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1060 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1061 }
1062 }
1063
1064 void ShenandoahConcurrentGC::op_class_unloading() {
1065 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1066 assert (heap->is_concurrent_weak_root_in_progress() &&
1067 heap->unload_classes(),
1068 "Checked by caller");
1069 heap->do_class_unloading();
1070 }
1071
1072 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1073 private:
1074 ShenandoahEvacuateUpdateMetadataClosure _cl;
1075
1076 public:
1077 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1078
1079 void do_nmethod(nmethod* n) {
1080 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1081 ShenandoahNMethodLocker locker(data->lock());
1082 ICacheInvalidationContext icic;
1083 data->oops_do(&_cl, /* fix_relocations = */ true, &icic);
1084 }
1085 };
1086
1087 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1088 private:
1089 ShenandoahPhaseTimings::Phase _phase;
1090 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1091 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1092 _cld_roots;
1093 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1094
1095 public:
1096 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1097 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1098 _phase(phase),
1099 _vm_roots(phase),
1100 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1101 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1102
1103 void work(uint worker_id) {
1104 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1105 {
1106 {
1107 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1108 // may race against OopStorage::release() calls.
1109 ShenandoahContextEvacuateUpdateRootsClosure cl;
1110 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1111 }
1112
1113 {
1114 ShenandoahEvacuateUpdateMetadataClosure cl;
1115 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1116 _cld_roots.cld_do(&clds, worker_id);
1117 }
1118 }
1119
1120 if (!ShenandoahHeap::heap()->unload_classes()) {
1121 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1122 ShenandoahEvacUpdateCodeCacheClosure cl;
1123 _nmethod_itr.nmethods_do(&cl);
1124 }
1125 }
1126 };
1127
1128 void ShenandoahConcurrentGC::op_strong_roots() {
1129 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1130 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1131 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1132 heap->workers()->run_task(&task);
1133 heap->set_concurrent_strong_root_in_progress(false);
1134 }
1135
1136 void ShenandoahConcurrentGC::op_cleanup_early() {
1137 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1138 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1139 "cleanup early.");
1140 ShenandoahHeap::heap()->recycle_trash();
1141 }
1142
1143 void ShenandoahConcurrentGC::op_evacuate() {
1144 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1145 }
1146
1147 void ShenandoahConcurrentGC::op_init_update_refs() {
1148 if (ShenandoahVerify) {
1149 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1150 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1151 heap->verifier()->verify_before_update_refs(_generation);
1152 }
1153 }
1154
1155 void ShenandoahConcurrentGC::op_update_refs() {
1156 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1157 }
1158
1159 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1160 private:
1161 // This closure runs when thread is stopped for handshake, which means
1162 // we can use non-concurrent closure here, as long as it only updates
1163 // locations modified by the thread itself, i.e. stack locations.
1164 ShenandoahNonConcUpdateRefsClosure _cl;
1165 public:
1166 ShenandoahUpdateThreadHandshakeClosure();
1167 void do_thread(Thread* thread) override;
1168 };
1169
1170 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1171 HandshakeClosure("Shenandoah Update Thread Roots") {
1172 }
1173
1174 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1175 if (thread->is_Java_thread()) {
1176 JavaThread* jt = JavaThread::cast(thread);
1177 ResourceMark rm;
1178 jt->oops_do(&_cl, nullptr);
1179 }
1180 }
1181
1182 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1183 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1184 // to pick up overwritten pointers that point into a cset region. If these pointers
1185 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1186 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1187 //
1188 // Therefore, at the end of update refs, we use this closure to update the thread roots
1189 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1190 // anything that has already been marked or anything that points to a region which is
1191 // not old. We do not need to worry about ABA situations where a region may become old
1192 // after the pointer is enqueued but before it is filtered. There are only two ways a
1193 // region may become old:
1194 // 1. The region is promoted in place. This is safe because such regions will never
1195 // be in the collection set. If this happens, the pointer will be preserved, essentially
1196 // becoming part of the old snapshot.
1197 // 2. The region is allocated during evacuation of old. This is also not a concern because
1198 // we haven't yet finished marking old so no mixed evacuations will happen.
1199 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1200 ShenandoahFlushSATB _flush_all_satb;
1201
1202 public:
1203 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1204 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1205 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1206 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1207 "Should be filtering pointers outside of old during old marking");
1208 }
1209
1210 void do_thread(Thread* thread) override {
1211 _update_roots.do_thread(thread);
1212 _flush_all_satb.do_thread(thread);
1213 }
1214 };
1215
1216 void ShenandoahConcurrentGC::op_update_thread_roots() {
1217 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1218 if (heap->is_concurrent_old_mark_in_progress()) {
1219 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1220 Handshake::execute(&cl);
1221 } else {
1222 ShenandoahUpdateThreadHandshakeClosure cl;
1223 Handshake::execute(&cl);
1224 }
1225 }
1226
1227 void ShenandoahConcurrentGC::op_final_update_refs() {
1228 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1229 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1230 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1231
1232 heap->finish_concurrent_roots();
1233
1234 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1235 // everything.
1236 if (heap->cancelled_gc()) {
1237 heap->clear_cancelled_gc();
1238 }
1239
1240 // Has to be done before cset is clear
1241 if (ShenandoahVerify) {
1242 heap->verifier()->verify_roots_in_to_space(_generation);
1243 }
1244
1245 // If we are running in generational mode, this will also age active regions that
1246 // haven't been used for allocation.
1247 heap->update_heap_region_states(true /*concurrent*/);
1248
1249 heap->set_update_refs_in_progress(false);
1250 heap->set_has_forwarded_objects(false);
1251
1252 if (ShenandoahVerify) {
1253 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1254 heap->verifier()->verify_after_update_refs(_generation);
1255 }
1256
1257 if (VerifyAfterGC) {
1258 Universe::verify();
1259 }
1260
1261 heap->rebuild_free_set(true /*concurrent*/);
1262 _generation->heuristics()->start_idle_span();
1263
1264 // Final pause: update GC barriers to idle state.
1265 CodeCache::arm_all_nmethods();
1266
1267 {
1268 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1269 heap->propagate_gc_state_to_all_threads();
1270 }
1271 }
1272
1273 void ShenandoahConcurrentGC::entry_final_roots() {
1274 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1275 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Roots", "");
1276 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1277 EventMark em("%s", msg);
1278
1279 heap->op_final_roots();
1280 }
1281
1282 void ShenandoahConcurrentGC::op_verify_final() {
1283 assert(ShenandoahVerify, "Should have been checked before");
1284 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1285 heap->verifier()->verify_after_gc(_generation);
1286 }
1287
1288 void ShenandoahConcurrentGC::op_cleanup_complete() {
1289 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1290 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1291 "cleanup complete.");
1292 ShenandoahHeap::heap()->recycle_trash();
1293 }
1294
1295 void ShenandoahConcurrentGC::op_reset_after_collect() {
1296 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1297 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1298 "reset after collection.");
1299
1300 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1301 if (heap->mode()->is_generational()) {
1302 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1303 // the young generation intact. In particular, reference processing in the old generation may potentially
1304 // need the reachability of a young generation referent of a Reference object in the old generation.
1305 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1306 heap->young_generation()->reset_mark_bitmap<false>();
1307 }
1308 } else {
1309 _generation->reset_mark_bitmap<false>();
1310 }
1311 }
1312
1313 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1314 if (ShenandoahHeap::heap()->cancelled_gc()) {
1315 _degen_point = point;
1316 return true;
1317 }
1318 return false;
1319 }