1 /*
2 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
36 #include "gc/shenandoah/shenandoahGeneration.hpp"
37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
38 #include "gc/shenandoah/shenandoahLock.hpp"
39 #include "gc/shenandoah/shenandoahMark.inline.hpp"
40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
41 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
42 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
45 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
46 #include "gc/shenandoah/shenandoahUtils.hpp"
47 #include "gc/shenandoah/shenandoahVerifier.hpp"
48 #include "gc/shenandoah/shenandoahVMOperations.hpp"
49 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
50 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
51 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
52 #include "memory/allocation.hpp"
53 #include "prims/jvmtiTagMap.hpp"
54 #include "runtime/vmThread.hpp"
55 #include "utilities/events.hpp"
56
57 // Breakpoint support
58 class ShenandoahBreakpointGCScope : public StackObj {
59 private:
60 const GCCause::Cause _cause;
61 public:
62 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
63 if (cause == GCCause::_wb_breakpoint) {
64 ShenandoahBreakpoint::start_gc();
65 ShenandoahBreakpoint::at_before_gc();
66 }
67 }
68
69 ~ShenandoahBreakpointGCScope() {
70 if (_cause == GCCause::_wb_breakpoint) {
71 ShenandoahBreakpoint::at_after_gc();
72 }
73 }
74 };
75
76 class ShenandoahBreakpointMarkScope : public StackObj {
77 private:
78 const GCCause::Cause _cause;
79 public:
80 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
81 if (_cause == GCCause::_wb_breakpoint) {
82 ShenandoahBreakpoint::at_after_marking_started();
83 }
84 }
85
86 ~ShenandoahBreakpointMarkScope() {
87 if (_cause == GCCause::_wb_breakpoint) {
88 ShenandoahBreakpoint::at_before_marking_completed();
89 }
90 }
91 };
92
93 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
94 ShenandoahGC(generation),
95 _mark(generation),
96 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
97 _abbreviated(false),
98 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
99 }
100
101 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
102 return _degen_point;
103 }
104
105 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
106 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
107 const char* msg = conc_init_update_refs_event_message();
108 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
109 EventMark em("%s", msg);
110
111 // Evacuation is complete, retire gc labs and change gc state
112 heap->concurrent_prepare_for_update_refs();
113 }
114
115 void ShenandoahConcurrentGC::entry_update_card_table() {
116 ShenandoahHeap* const heap = ShenandoahHeap::heap();
117 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
118
119 static const char* msg = "Concurrent update cards";
120 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
121 EventMark em("%s", msg);
122
123 ShenandoahWorkerScope scope(heap->workers(),
124 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
125 "concurrent update cards");
126
127 // Heap needs to be parsable here.
128 // Also, parallel heap region iterate must have a phase set.
129 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
130 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
131 }
132
133 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
134 ShenandoahHeap* const heap = ShenandoahHeap::heap();
135 _generation->ref_processor()->set_soft_reference_policy(
136 GCCause::should_clear_all_soft_refs(cause));
137
138 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
139
140 // Reset for upcoming marking
141 entry_reset();
142
143 // Start initial mark under STW
144 vmop_entry_init_mark();
145
146 {
147 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
148
149 // Reset task queue stats here, rather than in mark_concurrent_roots,
150 // because remembered set scan will `push` oops into the queues and
151 // resetting after this happens will lose those counts.
152 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
153
154 // Concurrent remembered set scanning
155 entry_scan_remembered_set();
156
157 // Concurrent mark roots
158 entry_mark_roots();
159 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
160 return false;
161 }
162
163 // Continue concurrent mark
164 entry_mark();
165 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
166 return false;
167 }
168 }
169
170 // Complete marking under STW, and start evacuation
171 vmop_entry_final_mark();
172
173 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
174 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
175 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
176 // from that phase.
177 if (_generation->is_concurrent_mark_in_progress()) {
178 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
179 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
180 return false;
181 }
182
183 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
184
185 // Concurrent stack processing
186 if (heap->is_evacuation_in_progress()) {
187 entry_thread_roots();
188 }
189
190 // Process weak roots that might still point to regions that would be broken by cleanup.
191 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
192 entry_weak_refs();
193 entry_weak_roots();
194
195 // Perform concurrent class unloading before any regions get recycled. Class unloading may
196 // need to inspect unmarked objects in trashed regions.
197 if (heap->unload_classes()) {
198 entry_class_unloading();
199 }
200
201 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
202 // the space. This would be the last action if there is nothing to evacuate. Note that
203 // we will not age young-gen objects in the case that we skip evacuation.
204 entry_cleanup_early();
205
206 heap->free_set()->log_status_under_lock();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Continue the cycle with evacuation and optional update-refs.
216 // This may be skipped if there is nothing to evacuate.
217 // If so, evac_in_progress would be unset by collection set preparation code.
218 if (heap->is_evacuation_in_progress()) {
219 // Concurrently evacuate
220 entry_evacuate();
221 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
222 return false;
223 }
224
225 // Perform update-refs phase.
226 entry_concurrent_update_refs_prepare(heap);
227
228 if (ShenandoahHeap::heap()->mode()->is_generational()) {
229 entry_update_card_table();
230 }
231
232 if (ShenandoahVerify) {
233 vmop_entry_init_update_refs();
234 }
235
236 entry_update_refs();
237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
238 return false;
239 }
240
241 // Concurrent update thread roots
242 entry_update_thread_roots();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 vmop_entry_final_update_refs();
248
249 // Update references freed up collection set, kick the cleanup to reclaim the space.
250 entry_cleanup_complete();
251 } else {
252 _abbreviated = true;
253
254 if (heap->mode()->is_generational()) {
255 if (!complete_abbreviated_cycle()) {
256 assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
257 return false;
258 }
259 }
260
261 // In normal cycle, final-update-refs would verify at the end of the cycle.
262 // In abbreviated cycle, we need to verify separately.
263 // This is now also puts the barriers down at the end of the cycle. TODO: Refine.
264 vmop_entry_final_roots();
265 }
266
267 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
268 // abbreviated cycle.
269 if (heap->mode()->is_generational()) {
270 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
271 }
272
273 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
274 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
275 // reducing the likelihood that GC will degenerate.
276 entry_reset_after_collect();
277
278 return true;
279 }
280
281 bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
282 shenandoah_assert_generational();
283
284 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
285
286 // We chose not to evacuate because we found sufficient immediate garbage.
287 // However, there may still be regions to promote in place, so do that now.
288 if (heap->old_generation()->has_in_place_promotions()) {
289 entry_promote_in_place();
290
291 // If the promote-in-place operation was cancelled, we can have the degenerated
292 // cycle complete the operation. It will see that no evacuations are in progress,
293 // and that there are regions wanting promotion. The risk with not handling the
294 // cancellation would be failing to restore top for these regions and leaving
295 // them unable to serve allocations for the old generation.This will leave the weak
296 // roots flag set (the degenerated cycle will unset it).
297 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
298 return false;
299 }
300 }
301
302 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
303 // the control thread will detect it on its next iteration and run a degenerated young cycle.
304 if (!_generation->is_old()) {
305 heap->update_region_ages(_generation->complete_marking_context());
306 }
307
308 return true;
309 }
310
311 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
312 ShenandoahHeap* const heap = ShenandoahHeap::heap();
313 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
314 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
315
316 heap->try_inject_alloc_failure();
317 VM_ShenandoahInitMark op(this);
318 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
319 }
320
321 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
322 ShenandoahHeap* const heap = ShenandoahHeap::heap();
323 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
324 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
325
326 heap->try_inject_alloc_failure();
327 VM_ShenandoahFinalMarkStartEvac op(this);
328 VMThread::execute(&op); // jump to entry_final_mark under safepoint
329 }
330
331 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
332 ShenandoahHeap* const heap = ShenandoahHeap::heap();
333 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
334 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
335
336 heap->try_inject_alloc_failure();
337 VM_ShenandoahInitUpdateRefs op(this);
338 VMThread::execute(&op);
339 }
340
341 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
342 ShenandoahHeap* const heap = ShenandoahHeap::heap();
343 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
344 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
345
346 heap->try_inject_alloc_failure();
347 VM_ShenandoahFinalUpdateRefs op(this);
348 VMThread::execute(&op);
349 }
350
351 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
352 ShenandoahHeap* const heap = ShenandoahHeap::heap();
353 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
354 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
355
356 // This phase does not use workers, no need for setup
357 heap->try_inject_alloc_failure();
358 VM_ShenandoahFinalRoots op(this);
359 VMThread::execute(&op);
360 }
361
362 void ShenandoahConcurrentGC::entry_init_mark() {
363 const char* msg = init_mark_event_message();
364 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
365 EventMark em("%s", msg);
366
367 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
368 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
369 "init marking");
370
371 op_init_mark();
372 }
373
374 void ShenandoahConcurrentGC::entry_final_mark() {
375 const char* msg = final_mark_event_message();
376 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
377 EventMark em("%s", msg);
378
379 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
380 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
381 "final marking");
382
383 op_final_mark();
384 }
385
386 void ShenandoahConcurrentGC::entry_init_update_refs() {
387 static const char* msg = "Pause Init Update Refs";
388 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
389 EventMark em("%s", msg);
390
391 // No workers used in this phase, no setup required
392 op_init_update_refs();
393 }
394
395 void ShenandoahConcurrentGC::entry_final_update_refs() {
396 static const char* msg = "Pause Final Update Refs";
397 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
398 EventMark em("%s", msg);
399
400 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
401 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
402 "final reference update");
403
404 op_final_update_refs();
405 }
406
407 void ShenandoahConcurrentGC::entry_reset() {
408 ShenandoahHeap* const heap = ShenandoahHeap::heap();
409 heap->try_inject_alloc_failure();
410
411 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
412 {
413 const char* msg = conc_reset_event_message();
414 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
415 EventMark em("%s", msg);
416
417 ShenandoahWorkerScope scope(heap->workers(),
418 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
419 msg);
420 op_reset();
421 }
422 }
423
424 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
425 if (_generation->is_young()) {
426 ShenandoahHeap* const heap = ShenandoahHeap::heap();
427 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
428 const char* msg = "Concurrent remembered set scanning";
429 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
430 EventMark em("%s", msg);
431
432 ShenandoahWorkerScope scope(heap->workers(),
433 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
434 msg);
435
436 heap->try_inject_alloc_failure();
437 _generation->scan_remembered_set(true /* is_concurrent */);
438 }
439 }
440
441 void ShenandoahConcurrentGC::entry_mark_roots() {
442 ShenandoahHeap* const heap = ShenandoahHeap::heap();
443 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
444 const char* msg = "Concurrent marking roots";
445 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
446 EventMark em("%s", msg);
447
448 ShenandoahWorkerScope scope(heap->workers(),
449 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
450 "concurrent marking roots");
451
452 heap->try_inject_alloc_failure();
453 op_mark_roots();
454 }
455
456 void ShenandoahConcurrentGC::entry_mark() {
457 ShenandoahHeap* const heap = ShenandoahHeap::heap();
458 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
459 const char* msg = conc_mark_event_message();
460 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
461 EventMark em("%s", msg);
462
463 ShenandoahWorkerScope scope(heap->workers(),
464 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
465 "concurrent marking");
466
467 heap->try_inject_alloc_failure();
468 op_mark();
469 }
470
471 void ShenandoahConcurrentGC::entry_thread_roots() {
472 ShenandoahHeap* const heap = ShenandoahHeap::heap();
473 static const char* msg = "Concurrent thread roots";
474 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
475 EventMark em("%s", msg);
476
477 ShenandoahWorkerScope scope(heap->workers(),
478 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
479 msg);
480
481 heap->try_inject_alloc_failure();
482 op_thread_roots();
483 }
484
485 void ShenandoahConcurrentGC::entry_weak_refs() {
486 ShenandoahHeap* const heap = ShenandoahHeap::heap();
487 const char* msg = conc_weak_refs_event_message();
488 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
489 EventMark em("%s", msg);
490
491 ShenandoahWorkerScope scope(heap->workers(),
492 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
493 "concurrent weak references");
494
495 heap->try_inject_alloc_failure();
496 op_weak_refs();
497 }
498
499 void ShenandoahConcurrentGC::entry_weak_roots() {
500 ShenandoahHeap* const heap = ShenandoahHeap::heap();
501 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
502 const char* msg = conc_weak_roots_event_message();
503 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
504 EventMark em("%s", msg);
505
506 ShenandoahWorkerScope scope(heap->workers(),
507 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
508 "concurrent weak root");
509
510 heap->try_inject_alloc_failure();
511 op_weak_roots();
512 }
513
514 void ShenandoahConcurrentGC::entry_class_unloading() {
515 ShenandoahHeap* const heap = ShenandoahHeap::heap();
516 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
517 static const char* msg = "Concurrent class unloading";
518 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
519 EventMark em("%s", msg);
520
521 ShenandoahWorkerScope scope(heap->workers(),
522 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
523 "concurrent class unloading");
524
525 heap->try_inject_alloc_failure();
526 op_class_unloading();
527 }
528
529 void ShenandoahConcurrentGC::entry_strong_roots() {
530 ShenandoahHeap* const heap = ShenandoahHeap::heap();
531 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
532 static const char* msg = "Concurrent strong roots";
533 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
534 EventMark em("%s", msg);
535
536 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
537
538 ShenandoahWorkerScope scope(heap->workers(),
539 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
540 "concurrent strong root");
541
542 heap->try_inject_alloc_failure();
543 op_strong_roots();
544 }
545
546 void ShenandoahConcurrentGC::entry_cleanup_early() {
547 ShenandoahHeap* const heap = ShenandoahHeap::heap();
548 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
549 const char* msg = conc_cleanup_event_message();
550 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
551 EventMark em("%s", msg);
552
553 // This phase does not use workers, no need for setup
554 heap->try_inject_alloc_failure();
555 op_cleanup_early();
556 if (!heap->is_evacuation_in_progress()) {
557 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
558 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
559 // during promote-in-place processing.
560 heap->rebuild_free_set(true /*concurrent*/);
561 }
562 }
563
564 void ShenandoahConcurrentGC::entry_evacuate() {
565 ShenandoahHeap* const heap = ShenandoahHeap::heap();
566 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
567
568 static const char* msg = "Concurrent evacuation";
569 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
570 EventMark em("%s", msg);
571
572 ShenandoahWorkerScope scope(heap->workers(),
573 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
574 "concurrent evacuation");
575
576 heap->try_inject_alloc_failure();
577 op_evacuate();
578 }
579
580 void ShenandoahConcurrentGC::entry_promote_in_place() const {
581 shenandoah_assert_generational();
582
583 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::promote_in_place);
584 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::promote_in_place);
585 EventMark em("%s", "Promote in place");
586
587 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(_generation, true);
588 }
589
590 void ShenandoahConcurrentGC::entry_update_thread_roots() {
591 ShenandoahHeap* const heap = ShenandoahHeap::heap();
592 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
593
594 static const char* msg = "Concurrent update thread roots";
595 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
596 EventMark em("%s", msg);
597
598 // No workers used in this phase, no setup required
599 heap->try_inject_alloc_failure();
600 op_update_thread_roots();
601 }
602
603 void ShenandoahConcurrentGC::entry_update_refs() {
604 ShenandoahHeap* const heap = ShenandoahHeap::heap();
605 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
606 static const char* msg = "Concurrent update references";
607 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
608 EventMark em("%s", msg);
609
610 ShenandoahWorkerScope scope(heap->workers(),
611 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
612 "concurrent reference update");
613
614 heap->try_inject_alloc_failure();
615 op_update_refs();
616 }
617
618 void ShenandoahConcurrentGC::entry_cleanup_complete() {
619 ShenandoahHeap* const heap = ShenandoahHeap::heap();
620 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
621 const char* msg = conc_cleanup_event_message();
622 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
623 EventMark em("%s", msg);
624
625 // This phase does not use workers, no need for setup
626 heap->try_inject_alloc_failure();
627 op_cleanup_complete();
628 }
629
630 void ShenandoahConcurrentGC::entry_reset_after_collect() {
631 ShenandoahHeap* const heap = ShenandoahHeap::heap();
632 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
633 const char* msg = conc_reset_after_collect_event_message();
634 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
635 EventMark em("%s", msg);
636
637 op_reset_after_collect();
638 }
639
640 void ShenandoahConcurrentGC::op_reset() {
641 ShenandoahHeap* const heap = ShenandoahHeap::heap();
642
643 // If it is old GC bootstrap cycle, always clear bitmap for global gen
644 // to ensure bitmap for old gen is clear for old GC cycle after this.
645 if (_do_old_gc_bootstrap) {
646 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
647 heap->global_generation()->prepare_gc();
648 } else {
649 _generation->prepare_gc();
650 }
651
652 if (heap->mode()->is_generational()) {
653 heap->old_generation()->card_scan()->mark_read_table_as_clean();
654 }
655 }
656
657 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
658 private:
659 ShenandoahMarkingContext* const _ctx;
660 public:
661 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
662
663 void heap_region_do(ShenandoahHeapRegion* r) {
664 assert(!r->has_live(), "Region %zu should have no live data", r->index());
665 if (r->is_active()) {
666 // Check if region needs updating its TAMS. We have updated it already during concurrent
667 // reset, so it is very likely we don't need to do another write here. Since most regions
668 // are not "active", this path is relatively rare.
669 if (_ctx->top_at_mark_start(r) != r->top()) {
670 _ctx->capture_top_at_mark_start(r);
671 }
672 } else {
673 assert(_ctx->top_at_mark_start(r) == r->top(),
674 "Region %zu should already have correct TAMS", r->index());
675 }
676 }
677
678 bool is_thread_safe() { return true; }
679 };
680
681 void ShenandoahConcurrentGC::start_mark() {
682 _mark.start_mark();
683 }
684
685 void ShenandoahConcurrentGC::op_init_mark() {
686 ShenandoahHeap* const heap = ShenandoahHeap::heap();
687 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
688 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
689
690 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
691 assert(!_generation->is_mark_complete(), "should not be complete");
692 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
693
694 // First pause in cycle, check that barriers were not left enabled.
695 ShenandoahCodeRoots::check_barriers();
696
697 if (heap->mode()->is_generational()) {
698 if (_generation->is_global()) {
699 heap->old_generation()->cancel_gc();
700 }
701
702 {
703 // After we swap card table below, the write-table is all clean, and the read table holds
704 // cards dirty prior to the start of GC. Young and bootstrap collection will update
705 // the write card table as a side effect of remembered set scanning. Global collection will
706 // update the card table as a side effect of global marking of old objects.
707 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
708 _generation->swap_card_tables();
709 }
710 }
711
712 if (ShenandoahVerify) {
713 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
714 heap->verifier()->verify_before_concmark(_generation);
715 }
716
717 if (VerifyBeforeGC) {
718 Universe::verify();
719 }
720
721 _generation->set_concurrent_mark_in_progress(true);
722
723 start_mark();
724
725 if (_do_old_gc_bootstrap) {
726 shenandoah_assert_generational();
727 // Update region state for both young and old regions
728 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
729 ShenandoahInitMarkUpdateRegionStateClosure cl;
730 heap->parallel_heap_region_iterate(&cl);
731 heap->old_generation()->ref_processor()->reset_thread_locals();
732 } else {
733 // Update region state for only young regions
734 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
735 ShenandoahInitMarkUpdateRegionStateClosure cl;
736 _generation->parallel_heap_region_iterate(&cl);
737 }
738
739 // Weak reference processing
740 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
741 rp->reset_thread_locals();
742
743 // Make above changes visible to worker threads
744 OrderAccess::fence();
745
746 // Arm nmethods/stack for concurrent processing
747 ShenandoahCodeRoots::arm_nmethods();
748 ShenandoahStackWatermark::change_epoch_id();
749
750 {
751 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
752 heap->propagate_gc_state_to_all_threads();
753 }
754 }
755
756 void ShenandoahConcurrentGC::op_mark_roots() {
757 _mark.mark_concurrent_roots();
758 }
759
760 void ShenandoahConcurrentGC::op_mark() {
761 _mark.concurrent_mark();
762 }
763
764 void ShenandoahConcurrentGC::op_final_mark() {
765 ShenandoahHeap* const heap = ShenandoahHeap::heap();
766 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
767 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
768
769 if (ShenandoahVerify) {
770 heap->verifier()->verify_roots_no_forwarded(_generation);
771 }
772
773 if (!heap->cancelled_gc()) {
774 _mark.finish_mark();
775 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
776
777 // Notify JVMTI that the tagmap table will need cleaning.
778 JvmtiTagMap::set_needs_cleaning();
779
780 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
781 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
782 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
783 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
784
785 // Has to be done after cset selection
786 heap->prepare_concurrent_roots();
787
788 if (!heap->collection_set()->is_empty()) {
789 LogTarget(Debug, gc, cset) lt;
790 if (lt.is_enabled()) {
791 ResourceMark rm;
792 LogStream ls(lt);
793 heap->collection_set()->print_on(&ls);
794 }
795
796 if (ShenandoahVerify) {
797 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
798 heap->verifier()->verify_before_evacuation(_generation);
799 }
800
801 heap->set_evacuation_in_progress(true);
802 // From here on, we need to update references.
803 heap->set_has_forwarded_objects(true);
804 } else {
805 if (ShenandoahVerify) {
806 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
807 if (has_in_place_promotions(heap)) {
808 heap->verifier()->verify_after_concmark_with_promotions(_generation);
809 } else {
810 heap->verifier()->verify_after_concmark(_generation);
811 }
812 }
813 }
814 }
815
816 // Arm nmethods/stack for concurrent processing
817 ShenandoahCodeRoots::arm_nmethods();
818 ShenandoahStackWatermark::change_epoch_id();
819
820 {
821 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
822 heap->propagate_gc_state_to_all_threads();
823 }
824 }
825
826 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
827 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
828 }
829
830 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
831 private:
832 OopClosure* const _oops;
833 public:
834 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
835
836 void do_thread(Thread* thread) override {
837 JavaThread* const jt = JavaThread::cast(thread);
838 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
839 }
840 };
841
842 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
843 private:
844 ShenandoahJavaThreadsIterator _java_threads;
845
846 public:
847 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
848 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
849 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
850 }
851
852 void work(uint worker_id) override {
853 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
854 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
855 _java_threads.threads_do(&thr_cl, worker_id);
856 }
857 };
858
859 void ShenandoahConcurrentGC::op_thread_roots() {
860 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
861 assert(heap->is_evacuation_in_progress(), "Checked by caller");
862 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
863 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
864 heap->workers()->run_task(&task);
865 }
866
867 void ShenandoahConcurrentGC::op_weak_refs() {
868 ShenandoahHeap* const heap = ShenandoahHeap::heap();
869 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
870 // Concurrent weak refs processing
871 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
872 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
873 ShenandoahBreakpoint::at_after_reference_processing_started();
874 }
875 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
876 }
877
878 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
879 private:
880 ShenandoahHeap* const _heap;
881 ShenandoahGeneration* const _generation;
882 ShenandoahMarkingContext* const _mark_context;
883 bool _evac_in_progress;
884 Thread* const _thread;
885
886 public:
887 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
888 void do_oop(oop* p);
889 void do_oop(narrowOop* p);
890 };
891
892 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
893 _heap(ShenandoahHeap::heap()),
894 _generation(generation),
895 _mark_context(ShenandoahHeap::heap()->marking_context()),
896 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
897 _thread(Thread::current()) {
898 }
899
900 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
901 const oop obj = RawAccess<>::oop_load(p);
902 if (!CompressedOops::is_null(obj)) {
903 if (!_mark_context->is_marked(obj)) {
904 if (_generation->contains(obj)) {
905 // Note: The obj is dead here. Do not touch it, just clear.
906 ShenandoahHeap::atomic_clear_oop(p, obj);
907 }
908 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
909 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
910 if (resolved == obj) {
911 resolved = _heap->evacuate_object(obj, _thread);
912 }
913 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
914 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
915 }
916 }
917 }
918
919 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
920 ShouldNotReachHere();
921 }
922
923 class ShenandoahIsCLDAliveClosure : public CLDClosure {
924 public:
925 void do_cld(ClassLoaderData* cld) {
926 cld->is_alive();
927 }
928 };
929
930 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
931 public:
932 void do_nmethod(nmethod* n) {
933 n->is_unloading();
934 }
935 };
936
937 // This task not only evacuates/updates marked weak roots, but also "null"
938 // dead weak roots.
939 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
940 private:
941 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
942
943 // Roots related to concurrent class unloading
944 ShenandoahClassLoaderDataRoots<true /* concurrent */>
945 _cld_roots;
946 ShenandoahConcurrentNMethodIterator _nmethod_itr;
947 ShenandoahGeneration* _generation;
948 ShenandoahPhaseTimings::Phase _phase;
949
950 public:
951 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
952 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
953 _vm_roots(phase),
954 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
955 _nmethod_itr(ShenandoahCodeRoots::table()),
956 _generation(generation),
957 _phase(phase) {}
958
959 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
960 // Notify runtime data structures of potentially dead oops
961 _vm_roots.report_num_dead();
962 }
963
964 void work(uint worker_id) override {
965 ShenandoahConcurrentWorkerSession worker_session(worker_id);
966 SuspendibleThreadSetJoiner sts_join;
967 {
968 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
969 // may race against OopStorage::release() calls.
970 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
971 _vm_roots.oops_do(&cl, worker_id);
972 }
973
974 // If we are going to perform concurrent class unloading later on, we need to
975 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
976 // can clean up immediate garbage sooner.
977 if (ShenandoahHeap::heap()->unload_classes()) {
978 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
979 // CLD's holder or evacuate it.
980 {
981 ShenandoahIsCLDAliveClosure is_cld_alive;
982 _cld_roots.cld_do(&is_cld_alive, worker_id);
983 }
984
985 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
986 // The closure calls nmethod->is_unloading(). The is_unloading
987 // state is cached, therefore, during concurrent class unloading phase,
988 // we will not touch the metadata of unloading nmethods
989 {
990 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
991 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
992 _nmethod_itr.nmethods_do(&is_nmethod_alive);
993 }
994 }
995 }
996 };
997
998 void ShenandoahConcurrentGC::op_weak_roots() {
999 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1000 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1001 {
1002 // Concurrent weak root processing
1003 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1004 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1005 heap->workers()->run_task(&task);
1006 }
1007
1008 {
1009 // It is possible for mutators executing the load reference barrier to have
1010 // loaded an oop through a weak handle that has since been nulled out by
1011 // weak root processing. Handshaking here forces them to complete the
1012 // barrier before the GC cycle continues and does something that would
1013 // change the evaluation of the barrier (for example, resetting the TAMS
1014 // on trashed regions could make an oop appear to be marked _after_ the
1015 // region has been recycled).
1016 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1017 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1018 }
1019 }
1020
1021 void ShenandoahConcurrentGC::op_class_unloading() {
1022 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1023 assert (heap->is_concurrent_weak_root_in_progress() &&
1024 heap->unload_classes(),
1025 "Checked by caller");
1026 heap->do_class_unloading();
1027 }
1028
1029 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1030 private:
1031 ShenandoahEvacuateUpdateMetadataClosure _cl;
1032
1033 public:
1034 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1035
1036 void do_nmethod(nmethod* n) {
1037 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1038 ShenandoahNMethodLocker locker(data->lock());
1039 data->oops_do(&_cl, /* fix_relocations = */ true);
1040 ShenandoahNMethod::disarm_nmethod_unlocked(n);
1041 }
1042 };
1043
1044 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1045 private:
1046 ShenandoahPhaseTimings::Phase _phase;
1047 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1048 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1049 _cld_roots;
1050 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1051
1052 public:
1053 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1054 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1055 _phase(phase),
1056 _vm_roots(phase),
1057 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1058 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1059
1060 void work(uint worker_id) {
1061 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1062 {
1063 {
1064 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1065 // may race against OopStorage::release() calls.
1066 ShenandoahContextEvacuateUpdateRootsClosure cl;
1067 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1068 }
1069
1070 {
1071 ShenandoahEvacuateUpdateMetadataClosure cl;
1072 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1073 _cld_roots.cld_do(&clds, worker_id);
1074 }
1075 }
1076
1077 if (!ShenandoahHeap::heap()->unload_classes()) {
1078 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1079 ShenandoahEvacUpdateCodeCacheClosure cl;
1080 _nmethod_itr.nmethods_do(&cl);
1081 }
1082 }
1083 };
1084
1085 void ShenandoahConcurrentGC::op_strong_roots() {
1086 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1087 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1088 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1089 heap->workers()->run_task(&task);
1090 heap->set_concurrent_strong_root_in_progress(false);
1091 }
1092
1093 void ShenandoahConcurrentGC::op_cleanup_early() {
1094 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1095 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1096 "cleanup early.");
1097 ShenandoahHeap::heap()->recycle_trash();
1098 }
1099
1100 void ShenandoahConcurrentGC::op_evacuate() {
1101 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1102 }
1103
1104 void ShenandoahConcurrentGC::op_init_update_refs() {
1105 if (ShenandoahVerify) {
1106 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1107 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1108 heap->verifier()->verify_before_update_refs(_generation);
1109 }
1110 }
1111
1112 void ShenandoahConcurrentGC::op_update_refs() {
1113 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1114 }
1115
1116 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1117 private:
1118 // This closure runs when thread is stopped for handshake, which means
1119 // we can use non-concurrent closure here, as long as it only updates
1120 // locations modified by the thread itself, i.e. stack locations.
1121 ShenandoahNonConcUpdateRefsClosure _cl;
1122 public:
1123 ShenandoahUpdateThreadHandshakeClosure();
1124 void do_thread(Thread* thread) override;
1125 };
1126
1127 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1128 HandshakeClosure("Shenandoah Update Thread Roots") {
1129 }
1130
1131 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1132 if (thread->is_Java_thread()) {
1133 JavaThread* jt = JavaThread::cast(thread);
1134 ResourceMark rm;
1135 jt->oops_do(&_cl, nullptr);
1136 }
1137 }
1138
1139 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1140 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1141 // to pick up overwritten pointers that point into a cset region. If these pointers
1142 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1143 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1144 //
1145 // Therefore, at the end of update refs, we use this closure to update the thread roots
1146 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1147 // anything that has already been marked or anything that points to a region which is
1148 // not old. We do not need to worry about ABA situations where a region may become old
1149 // after the pointer is enqueued but before it is filtered. There are only two ways a
1150 // region may become old:
1151 // 1. The region is promoted in place. This is safe because such regions will never
1152 // be in the collection set. If this happens, the pointer will be preserved, essentially
1153 // becoming part of the old snapshot.
1154 // 2. The region is allocated during evacuation of old. This is also not a concern because
1155 // we haven't yet finished marking old so no mixed evacuations will happen.
1156 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1157 ShenandoahFlushSATB _flush_all_satb;
1158
1159 public:
1160 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1161 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1162 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1163 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1164 "Should be filtering pointers outside of old during old marking");
1165 }
1166
1167 void do_thread(Thread* thread) override {
1168 _update_roots.do_thread(thread);
1169 _flush_all_satb.do_thread(thread);
1170 }
1171 };
1172
1173 void ShenandoahConcurrentGC::op_update_thread_roots() {
1174 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1175 if (heap->is_concurrent_old_mark_in_progress()) {
1176 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1177 Handshake::execute(&cl);
1178 } else {
1179 ShenandoahUpdateThreadHandshakeClosure cl;
1180 Handshake::execute(&cl);
1181 }
1182 }
1183
1184 void ShenandoahConcurrentGC::op_final_update_refs() {
1185 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1186 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1187 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1188
1189 heap->finish_concurrent_roots();
1190
1191 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1192 // everything.
1193 if (heap->cancelled_gc()) {
1194 heap->clear_cancelled_gc();
1195 }
1196
1197 // Has to be done before cset is clear
1198 if (ShenandoahVerify) {
1199 heap->verifier()->verify_roots_in_to_space(_generation);
1200 }
1201
1202 // If we are running in generational mode and this is an aging cycle, this will also age active
1203 // regions that haven't been used for allocation.
1204 heap->update_heap_region_states(true /*concurrent*/);
1205
1206 heap->set_update_refs_in_progress(false);
1207 heap->set_has_forwarded_objects(false);
1208
1209 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1210 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1211 // entire regions. Both of these relevant operations occur before final update refs.
1212 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1213 }
1214
1215 if (ShenandoahVerify) {
1216 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1217 heap->verifier()->verify_after_update_refs(_generation);
1218 }
1219
1220 if (VerifyAfterGC) {
1221 Universe::verify();
1222 }
1223
1224 heap->rebuild_free_set(true /*concurrent*/);
1225 _generation->heuristics()->start_idle_span();
1226
1227 {
1228 // Final pause: update GC barriers to idle state.
1229 ShenandoahCodeRoots::arm_nmethods();
1230 ShenandoahStackWatermark::change_epoch_id();
1231 }
1232
1233 {
1234 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1235 heap->propagate_gc_state_to_all_threads();
1236 }
1237 }
1238
1239 void ShenandoahConcurrentGC::entry_final_roots() {
1240 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1241 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1242
1243 const char* msg = final_roots_event_message();
1244 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1245 EventMark em("%s", msg);
1246 ShenandoahWorkerScope scope(heap->workers(),
1247 ParallelGCThreads,
1248 msg);
1249
1250 heap->op_final_roots();
1251 }
1252
1253 void ShenandoahConcurrentGC::op_cleanup_complete() {
1254 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1255 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1256 "cleanup complete.");
1257 ShenandoahHeap::heap()->recycle_trash();
1258 }
1259
1260 void ShenandoahConcurrentGC::op_reset_after_collect() {
1261 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1262 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1263 "reset after collection.");
1264
1265 // Final concurrent phase: complete disabling all barriers.
1266 ShenandoahCodeRoots::disarm_nmethods();
1267
1268 // Check that barriers were not left enabled.
1269 ShenandoahCodeRoots::check_barriers();
1270
1271 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1272 if (heap->mode()->is_generational()) {
1273 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1274 // the young generation intact. In particular, reference processing in the old generation may potentially
1275 // need the reachability of a young generation referent of a Reference object in the old generation.
1276 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1277 heap->young_generation()->reset_mark_bitmap<false>();
1278 }
1279 } else {
1280 _generation->reset_mark_bitmap<false>();
1281 }
1282 }
1283
1284 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1285 if (ShenandoahHeap::heap()->cancelled_gc()) {
1286 _degen_point = point;
1287 return true;
1288 }
1289 return false;
1290 }
1291
1292 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1293 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1294 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1295 if (heap->unload_classes()) {
1296 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1297 } else {
1298 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1299 }
1300 }
1301
1302 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1303 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1304 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1305 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1306
1307 if (heap->unload_classes()) {
1308 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1309 } else {
1310 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1311 }
1312 }
1313
1314 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1315 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1316 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1317 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1318 if (heap->unload_classes()) {
1319 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1320 } else {
1321 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1322 }
1323 }
1324
1325 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1326 if (ShenandoahHeap::heap()->unload_classes()) {
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1328 } else {
1329 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1330 }
1331 }
1332
1333 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1334 if (ShenandoahHeap::heap()->unload_classes()) {
1335 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1336 } else {
1337 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1338 }
1339 }
1340
1341 const char* ShenandoahConcurrentGC::final_roots_event_message() const {
1342 if (ShenandoahHeap::heap()->unload_classes()) {
1343 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
1344 } else {
1345 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
1346 }
1347 }
1348
1349 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1350 if (ShenandoahHeap::heap()->unload_classes()) {
1351 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1352 } else {
1353 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1354 }
1355 }
1356
1357 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1358 if (ShenandoahHeap::heap()->unload_classes()) {
1359 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1360 } else {
1361 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1362 }
1363 }
1364
1365 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1366 if (ShenandoahHeap::heap()->unload_classes()) {
1367 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1368 } else {
1369 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1370 }
1371 }
1372
1373 const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
1374 if (ShenandoahHeap::heap()->unload_classes()) {
1375 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
1376 } else {
1377 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
1378 }
1379 }