1 /*
2 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
36 #include "gc/shenandoah/shenandoahGeneration.hpp"
37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
38 #include "gc/shenandoah/shenandoahLock.hpp"
39 #include "gc/shenandoah/shenandoahMark.inline.hpp"
40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
41 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
42 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
45 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
46 #include "gc/shenandoah/shenandoahUtils.hpp"
47 #include "gc/shenandoah/shenandoahVerifier.hpp"
48 #include "gc/shenandoah/shenandoahVMOperations.hpp"
49 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
50 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
51 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
52 #include "memory/allocation.hpp"
53 #include "prims/jvmtiTagMap.hpp"
54 #include "runtime/vmThread.hpp"
55 #include "utilities/events.hpp"
56
57 // Breakpoint support
58 class ShenandoahBreakpointGCScope : public StackObj {
59 private:
60 const GCCause::Cause _cause;
61 public:
62 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
63 if (cause == GCCause::_wb_breakpoint) {
64 ShenandoahBreakpoint::start_gc();
65 ShenandoahBreakpoint::at_before_gc();
66 }
67 }
68
69 ~ShenandoahBreakpointGCScope() {
70 if (_cause == GCCause::_wb_breakpoint) {
71 ShenandoahBreakpoint::at_after_gc();
72 }
73 }
74 };
75
76 class ShenandoahBreakpointMarkScope : public StackObj {
77 private:
78 const GCCause::Cause _cause;
79 public:
80 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
81 if (_cause == GCCause::_wb_breakpoint) {
82 ShenandoahBreakpoint::at_after_marking_started();
83 }
84 }
85
86 ~ShenandoahBreakpointMarkScope() {
87 if (_cause == GCCause::_wb_breakpoint) {
88 ShenandoahBreakpoint::at_before_marking_completed();
89 }
90 }
91 };
92
93 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
94 ShenandoahGC(generation),
95 _mark(generation),
96 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
97 _abbreviated(false),
98 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
99 }
100
101 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
102 return _degen_point;
103 }
104
105 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
106 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
107 const char* msg = conc_init_update_refs_event_message();
108 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
109 EventMark em("%s", msg);
110
111 // Evacuation is complete, retire gc labs and change gc state
112 heap->concurrent_prepare_for_update_refs();
113 }
114
115 void ShenandoahConcurrentGC::entry_update_card_table() {
116 ShenandoahHeap* const heap = ShenandoahHeap::heap();
117 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
118
119 static const char* msg = "Concurrent update cards";
120 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
121 EventMark em("%s", msg);
122
123 ShenandoahWorkerScope scope(heap->workers(),
124 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
125 "concurrent update cards");
126
127 // Heap needs to be parsable here.
128 // Also, parallel heap region iterate must have a phase set.
129 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
130 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
131 }
132
133 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
134 ShenandoahHeap* const heap = ShenandoahHeap::heap();
135 _generation->ref_processor()->set_soft_reference_policy(
136 GCCause::should_clear_all_soft_refs(cause));
137
138 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
139
140 // Reset for upcoming marking
141 entry_reset();
142
143 // Start initial mark under STW
144 vmop_entry_init_mark();
145
146 {
147 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
148
149 // Reset task queue stats here, rather than in mark_concurrent_roots,
150 // because remembered set scan will `push` oops into the queues and
151 // resetting after this happens will lose those counts.
152 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
153
154 // Concurrent remembered set scanning
155 entry_scan_remembered_set();
156
157 // Concurrent mark roots
158 entry_mark_roots();
159 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
160 return false;
161 }
162
163 // Continue concurrent mark
164 entry_mark();
165 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
166 return false;
167 }
168 }
169
170 // Complete marking under STW, and start evacuation
171 vmop_entry_final_mark();
172
173 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
174 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
175 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
176 // from that phase.
177 if (_generation->is_concurrent_mark_in_progress()) {
178 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
179 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
180 return false;
181 }
182
183 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
184
185 // Concurrent stack processing
186 if (heap->is_evacuation_in_progress()) {
187 entry_thread_roots();
188 }
189
190 // Process weak roots that might still point to regions that would be broken by cleanup.
191 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
192 entry_weak_refs();
193 entry_weak_roots();
194
195 // Perform concurrent class unloading before any regions get recycled. Class unloading may
196 // need to inspect unmarked objects in trashed regions.
197 if (heap->unload_classes()) {
198 entry_class_unloading();
199 }
200
201 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
202 // the space. This would be the last action if there is nothing to evacuate. Note that
203 // we will not age young-gen objects in the case that we skip evacuation.
204 entry_cleanup_early();
205
206 heap->free_set()->log_status_under_lock();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Continue the cycle with evacuation and optional update-refs.
216 // This may be skipped if there is nothing to evacuate.
217 // If so, evac_in_progress would be unset by collection set preparation code.
218 if (heap->is_evacuation_in_progress()) {
219 // Concurrently evacuate
220 entry_evacuate();
221 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
222 return false;
223 }
224
225 // Perform update-refs phase.
226 entry_concurrent_update_refs_prepare(heap);
227
228 if (ShenandoahHeap::heap()->mode()->is_generational()) {
229 entry_update_card_table();
230 }
231
232 if (ShenandoahVerify) {
233 vmop_entry_init_update_refs();
234 }
235
236 entry_update_refs();
237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
238 return false;
239 }
240
241 // Concurrent update thread roots
242 entry_update_thread_roots();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 vmop_entry_final_update_refs();
248
249 // Update references freed up collection set, kick the cleanup to reclaim the space.
250 entry_cleanup_complete();
251 } else {
252 _abbreviated = true;
253 if (!entry_final_roots()) {
254 assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
255 return false;
256 }
257
258 // In normal cycle, final-update-refs would verify at the end of the cycle.
259 // In abbreviated cycle, we need to verify separately.
260 if (ShenandoahVerify) {
261 vmop_entry_final_verify();
262 }
263 }
264
265 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
266 // abbreviated cycle.
267 if (heap->mode()->is_generational()) {
268 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
269 }
270
271 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
272 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
273 // reducing the likelihood that GC will degenerate.
274 entry_reset_after_collect();
275
276 return true;
277 }
278
279 bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
280 shenandoah_assert_generational();
281
282 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
283
284 // We chose not to evacuate because we found sufficient immediate garbage.
285 // However, there may still be regions to promote in place, so do that now.
286 if (heap->old_generation()->has_in_place_promotions()) {
287 entry_promote_in_place();
288
289 // If the promote-in-place operation was cancelled, we can have the degenerated
290 // cycle complete the operation. It will see that no evacuations are in progress,
291 // and that there are regions wanting promotion. The risk with not handling the
292 // cancellation would be failing to restore top for these regions and leaving
293 // them unable to serve allocations for the old generation.This will leave the weak
294 // roots flag set (the degenerated cycle will unset it).
295 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
296 return false;
297 }
298 }
299
300 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
301 // the control thread will detect it on its next iteration and run a degenerated young cycle.
302 if (!_generation->is_old()) {
303 heap->update_region_ages(_generation->complete_marking_context());
304 }
305
306 return true;
307 }
308
309 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
310 ShenandoahHeap* const heap = ShenandoahHeap::heap();
311 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
312 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
313
314 heap->try_inject_alloc_failure();
315 VM_ShenandoahInitMark op(this);
316 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
317 }
318
319 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
320 ShenandoahHeap* const heap = ShenandoahHeap::heap();
321 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
322 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
323
324 heap->try_inject_alloc_failure();
325 VM_ShenandoahFinalMarkStartEvac op(this);
326 VMThread::execute(&op); // jump to entry_final_mark under safepoint
327 }
328
329 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
330 ShenandoahHeap* const heap = ShenandoahHeap::heap();
331 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
332 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
333
334 heap->try_inject_alloc_failure();
335 VM_ShenandoahInitUpdateRefs op(this);
336 VMThread::execute(&op);
337 }
338
339 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
340 ShenandoahHeap* const heap = ShenandoahHeap::heap();
341 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
342 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
343
344 heap->try_inject_alloc_failure();
345 VM_ShenandoahFinalUpdateRefs op(this);
346 VMThread::execute(&op);
347 }
348
349 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
350 ShenandoahHeap* const heap = ShenandoahHeap::heap();
351 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
352 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
353
354 // This phase does not use workers, no need for setup
355 heap->try_inject_alloc_failure();
356 VM_ShenandoahFinalVerify op(this);
357 VMThread::execute(&op);
358 }
359
360 void ShenandoahConcurrentGC::entry_init_mark() {
361 const char* msg = init_mark_event_message();
362 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
363 EventMark em("%s", msg);
364
365 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
366 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
367 "init marking");
368
369 op_init_mark();
370 }
371
372 void ShenandoahConcurrentGC::entry_final_mark() {
373 const char* msg = final_mark_event_message();
374 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
375 EventMark em("%s", msg);
376
377 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
378 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
379 "final marking");
380
381 op_final_mark();
382 }
383
384 void ShenandoahConcurrentGC::entry_init_update_refs() {
385 static const char* msg = "Pause Init Update Refs";
386 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
387 EventMark em("%s", msg);
388
389 // No workers used in this phase, no setup required
390 op_init_update_refs();
391 }
392
393 void ShenandoahConcurrentGC::entry_final_update_refs() {
394 static const char* msg = "Pause Final Update Refs";
395 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
396 EventMark em("%s", msg);
397
398 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
399 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
400 "final reference update");
401
402 op_final_update_refs();
403 }
404
405 void ShenandoahConcurrentGC::entry_final_verify() {
406 const char* msg = verify_final_event_message();
407 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
408 EventMark em("%s", msg);
409
410 op_verify_final();
411 }
412
413 void ShenandoahConcurrentGC::entry_reset() {
414 ShenandoahHeap* const heap = ShenandoahHeap::heap();
415 heap->try_inject_alloc_failure();
416
417 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
418 {
419 const char* msg = conc_reset_event_message();
420 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
421 EventMark em("%s", msg);
422
423 ShenandoahWorkerScope scope(heap->workers(),
424 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
425 msg);
426 op_reset();
427 }
428 }
429
430 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
431 if (_generation->is_young()) {
432 ShenandoahHeap* const heap = ShenandoahHeap::heap();
433 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
434 const char* msg = "Concurrent remembered set scanning";
435 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
436 EventMark em("%s", msg);
437
438 ShenandoahWorkerScope scope(heap->workers(),
439 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
440 msg);
441
442 heap->try_inject_alloc_failure();
443 _generation->scan_remembered_set(true /* is_concurrent */);
444 }
445 }
446
447 void ShenandoahConcurrentGC::entry_mark_roots() {
448 ShenandoahHeap* const heap = ShenandoahHeap::heap();
449 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
450 const char* msg = "Concurrent marking roots";
451 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
452 EventMark em("%s", msg);
453
454 ShenandoahWorkerScope scope(heap->workers(),
455 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
456 "concurrent marking roots");
457
458 heap->try_inject_alloc_failure();
459 op_mark_roots();
460 }
461
462 void ShenandoahConcurrentGC::entry_mark() {
463 ShenandoahHeap* const heap = ShenandoahHeap::heap();
464 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
465 const char* msg = conc_mark_event_message();
466 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
467 EventMark em("%s", msg);
468
469 ShenandoahWorkerScope scope(heap->workers(),
470 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
471 "concurrent marking");
472
473 heap->try_inject_alloc_failure();
474 op_mark();
475 }
476
477 void ShenandoahConcurrentGC::entry_thread_roots() {
478 ShenandoahHeap* const heap = ShenandoahHeap::heap();
479 static const char* msg = "Concurrent thread roots";
480 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
481 EventMark em("%s", msg);
482
483 ShenandoahWorkerScope scope(heap->workers(),
484 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
485 msg);
486
487 heap->try_inject_alloc_failure();
488 op_thread_roots();
489 }
490
491 void ShenandoahConcurrentGC::entry_weak_refs() {
492 ShenandoahHeap* const heap = ShenandoahHeap::heap();
493 const char* msg = conc_weak_refs_event_message();
494 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
495 EventMark em("%s", msg);
496
497 ShenandoahWorkerScope scope(heap->workers(),
498 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
499 "concurrent weak references");
500
501 heap->try_inject_alloc_failure();
502 op_weak_refs();
503 }
504
505 void ShenandoahConcurrentGC::entry_weak_roots() {
506 ShenandoahHeap* const heap = ShenandoahHeap::heap();
507 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
508 const char* msg = conc_weak_roots_event_message();
509 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
510 EventMark em("%s", msg);
511
512 ShenandoahWorkerScope scope(heap->workers(),
513 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
514 "concurrent weak root");
515
516 heap->try_inject_alloc_failure();
517 op_weak_roots();
518 }
519
520 void ShenandoahConcurrentGC::entry_class_unloading() {
521 ShenandoahHeap* const heap = ShenandoahHeap::heap();
522 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
523 static const char* msg = "Concurrent class unloading";
524 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
525 EventMark em("%s", msg);
526
527 ShenandoahWorkerScope scope(heap->workers(),
528 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
529 "concurrent class unloading");
530
531 heap->try_inject_alloc_failure();
532 op_class_unloading();
533 }
534
535 void ShenandoahConcurrentGC::entry_strong_roots() {
536 ShenandoahHeap* const heap = ShenandoahHeap::heap();
537 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
538 static const char* msg = "Concurrent strong roots";
539 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
540 EventMark em("%s", msg);
541
542 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
543
544 ShenandoahWorkerScope scope(heap->workers(),
545 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
546 "concurrent strong root");
547
548 heap->try_inject_alloc_failure();
549 op_strong_roots();
550 }
551
552 void ShenandoahConcurrentGC::entry_cleanup_early() {
553 ShenandoahHeap* const heap = ShenandoahHeap::heap();
554 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
555 const char* msg = conc_cleanup_event_message();
556 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
557 EventMark em("%s", msg);
558
559 // This phase does not use workers, no need for setup
560 heap->try_inject_alloc_failure();
561 op_cleanup_early();
562 if (!heap->is_evacuation_in_progress()) {
563 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
564 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
565 // during promote-in-place processing.
566 heap->rebuild_free_set(true /*concurrent*/);
567 }
568 }
569
570 void ShenandoahConcurrentGC::entry_evacuate() {
571 ShenandoahHeap* const heap = ShenandoahHeap::heap();
572 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
573
574 static const char* msg = "Concurrent evacuation";
575 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
576 EventMark em("%s", msg);
577
578 ShenandoahWorkerScope scope(heap->workers(),
579 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
580 "concurrent evacuation");
581
582 heap->try_inject_alloc_failure();
583 op_evacuate();
584 }
585
586 void ShenandoahConcurrentGC::entry_promote_in_place() const {
587 shenandoah_assert_generational();
588
589 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::promote_in_place);
590 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::promote_in_place);
591 EventMark em("%s", "Promote in place");
592
593 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(_generation, true);
594 }
595
596 void ShenandoahConcurrentGC::entry_update_thread_roots() {
597 ShenandoahHeap* const heap = ShenandoahHeap::heap();
598 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
599
600 static const char* msg = "Concurrent update thread roots";
601 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
602 EventMark em("%s", msg);
603
604 // No workers used in this phase, no setup required
605 heap->try_inject_alloc_failure();
606 op_update_thread_roots();
607 }
608
609 void ShenandoahConcurrentGC::entry_update_refs() {
610 ShenandoahHeap* const heap = ShenandoahHeap::heap();
611 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
612 static const char* msg = "Concurrent update references";
613 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
614 EventMark em("%s", msg);
615
616 ShenandoahWorkerScope scope(heap->workers(),
617 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
618 "concurrent reference update");
619
620 heap->try_inject_alloc_failure();
621 op_update_refs();
622 }
623
624 void ShenandoahConcurrentGC::entry_cleanup_complete() {
625 ShenandoahHeap* const heap = ShenandoahHeap::heap();
626 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
627 const char* msg = conc_cleanup_event_message();
628 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
629 EventMark em("%s", msg);
630
631 // This phase does not use workers, no need for setup
632 heap->try_inject_alloc_failure();
633 op_cleanup_complete();
634 }
635
636 void ShenandoahConcurrentGC::entry_reset_after_collect() {
637 ShenandoahHeap* const heap = ShenandoahHeap::heap();
638 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
639 const char* msg = conc_reset_after_collect_event_message();
640 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
641 EventMark em("%s", msg);
642
643 op_reset_after_collect();
644 }
645
646 void ShenandoahConcurrentGC::op_reset() {
647 ShenandoahHeap* const heap = ShenandoahHeap::heap();
648
649 // If it is old GC bootstrap cycle, always clear bitmap for global gen
650 // to ensure bitmap for old gen is clear for old GC cycle after this.
651 if (_do_old_gc_bootstrap) {
652 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
653 heap->global_generation()->prepare_gc();
654 } else {
655 _generation->prepare_gc();
656 }
657
658 if (heap->mode()->is_generational()) {
659 heap->old_generation()->card_scan()->mark_read_table_as_clean();
660 }
661 }
662
663 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
664 private:
665 ShenandoahMarkingContext* const _ctx;
666 public:
667 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
668
669 void heap_region_do(ShenandoahHeapRegion* r) {
670 assert(!r->has_live(), "Region %zu should have no live data", r->index());
671 if (r->is_active()) {
672 // Check if region needs updating its TAMS. We have updated it already during concurrent
673 // reset, so it is very likely we don't need to do another write here. Since most regions
674 // are not "active", this path is relatively rare.
675 if (_ctx->top_at_mark_start(r) != r->top()) {
676 _ctx->capture_top_at_mark_start(r);
677 }
678 } else {
679 assert(_ctx->top_at_mark_start(r) == r->top(),
680 "Region %zu should already have correct TAMS", r->index());
681 }
682 }
683
684 bool is_thread_safe() { return true; }
685 };
686
687 void ShenandoahConcurrentGC::start_mark() {
688 _mark.start_mark();
689 }
690
691 void ShenandoahConcurrentGC::op_init_mark() {
692 ShenandoahHeap* const heap = ShenandoahHeap::heap();
693 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
694 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
695
696 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
697 assert(!_generation->is_mark_complete(), "should not be complete");
698 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
699
700 if (heap->mode()->is_generational()) {
701 if (_generation->is_global()) {
702 heap->old_generation()->cancel_gc();
703 }
704
705 {
706 // After we swap card table below, the write-table is all clean, and the read table holds
707 // cards dirty prior to the start of GC. Young and bootstrap collection will update
708 // the write card table as a side effect of remembered set scanning. Global collection will
709 // update the card table as a side effect of global marking of old objects.
710 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
711 _generation->swap_card_tables();
712 }
713 }
714
715 if (ShenandoahVerify) {
716 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
717 heap->verifier()->verify_before_concmark(_generation);
718 }
719
720 if (VerifyBeforeGC) {
721 Universe::verify();
722 }
723
724 _generation->set_concurrent_mark_in_progress(true);
725
726 start_mark();
727
728 if (_do_old_gc_bootstrap) {
729 shenandoah_assert_generational();
730 // Update region state for both young and old regions
731 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
732 ShenandoahInitMarkUpdateRegionStateClosure cl;
733 heap->parallel_heap_region_iterate(&cl);
734 heap->old_generation()->ref_processor()->reset_thread_locals();
735 } else {
736 // Update region state for only young regions
737 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
738 ShenandoahInitMarkUpdateRegionStateClosure cl;
739 _generation->parallel_heap_region_iterate(&cl);
740 }
741
742 // Weak reference processing
743 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
744 rp->reset_thread_locals();
745
746 // Make above changes visible to worker threads
747 OrderAccess::fence();
748
749 // Arm nmethods/stack for concurrent processing
750 ShenandoahCodeRoots::arm_nmethods();
751 ShenandoahStackWatermark::change_epoch_id();
752
753 {
754 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
755 heap->propagate_gc_state_to_all_threads();
756 }
757 }
758
759 void ShenandoahConcurrentGC::op_mark_roots() {
760 _mark.mark_concurrent_roots();
761 }
762
763 void ShenandoahConcurrentGC::op_mark() {
764 _mark.concurrent_mark();
765 }
766
767 void ShenandoahConcurrentGC::op_final_mark() {
768 ShenandoahHeap* const heap = ShenandoahHeap::heap();
769 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
770 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
771
772 if (ShenandoahVerify) {
773 heap->verifier()->verify_roots_no_forwarded(_generation);
774 }
775
776 if (!heap->cancelled_gc()) {
777 _mark.finish_mark();
778 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
779
780 // Notify JVMTI that the tagmap table will need cleaning.
781 JvmtiTagMap::set_needs_cleaning();
782
783 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
784 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
785 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
786 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
787
788 // Has to be done after cset selection
789 heap->prepare_concurrent_roots();
790
791 if (!heap->collection_set()->is_empty()) {
792 LogTarget(Debug, gc, cset) lt;
793 if (lt.is_enabled()) {
794 ResourceMark rm;
795 LogStream ls(lt);
796 heap->collection_set()->print_on(&ls);
797 }
798
799 if (ShenandoahVerify) {
800 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
801 heap->verifier()->verify_before_evacuation(_generation);
802 }
803
804 heap->set_evacuation_in_progress(true);
805 // From here on, we need to update references.
806 heap->set_has_forwarded_objects(true);
807
808 // Arm nmethods/stack for concurrent processing
809 ShenandoahCodeRoots::arm_nmethods();
810 ShenandoahStackWatermark::change_epoch_id();
811
812 } else {
813 if (ShenandoahVerify) {
814 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
815 if (has_in_place_promotions(heap)) {
816 heap->verifier()->verify_after_concmark_with_promotions(_generation);
817 } else {
818 heap->verifier()->verify_after_concmark(_generation);
819 }
820 }
821 }
822 }
823
824 {
825 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
826 heap->propagate_gc_state_to_all_threads();
827 }
828 }
829
830 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
831 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
832 }
833
834 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
835 private:
836 OopClosure* const _oops;
837 public:
838 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
839
840 void do_thread(Thread* thread) override {
841 JavaThread* const jt = JavaThread::cast(thread);
842 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
843 }
844 };
845
846 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
847 private:
848 ShenandoahJavaThreadsIterator _java_threads;
849
850 public:
851 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
852 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
853 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
854 }
855
856 void work(uint worker_id) override {
857 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
858 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
859 _java_threads.threads_do(&thr_cl, worker_id);
860 }
861 };
862
863 void ShenandoahConcurrentGC::op_thread_roots() {
864 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
865 assert(heap->is_evacuation_in_progress(), "Checked by caller");
866 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
867 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
868 heap->workers()->run_task(&task);
869 }
870
871 void ShenandoahConcurrentGC::op_weak_refs() {
872 ShenandoahHeap* const heap = ShenandoahHeap::heap();
873 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
874 // Concurrent weak refs processing
875 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
876 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
877 ShenandoahBreakpoint::at_after_reference_processing_started();
878 }
879 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
880 }
881
882 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
883 private:
884 ShenandoahHeap* const _heap;
885 ShenandoahGeneration* const _generation;
886 ShenandoahMarkingContext* const _mark_context;
887 bool _evac_in_progress;
888 Thread* const _thread;
889
890 public:
891 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
892 void do_oop(oop* p);
893 void do_oop(narrowOop* p);
894 };
895
896 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
897 _heap(ShenandoahHeap::heap()),
898 _generation(generation),
899 _mark_context(ShenandoahHeap::heap()->marking_context()),
900 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
901 _thread(Thread::current()) {
902 }
903
904 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
905 const oop obj = RawAccess<>::oop_load(p);
906 if (!CompressedOops::is_null(obj)) {
907 if (!_mark_context->is_marked(obj)) {
908 if (_generation->contains(obj)) {
909 // Note: The obj is dead here. Do not touch it, just clear.
910 ShenandoahHeap::atomic_clear_oop(p, obj);
911 }
912 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
913 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
914 if (resolved == obj) {
915 resolved = _heap->evacuate_object(obj, _thread);
916 }
917 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
918 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
919 }
920 }
921 }
922
923 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
924 ShouldNotReachHere();
925 }
926
927 class ShenandoahIsCLDAliveClosure : public CLDClosure {
928 public:
929 void do_cld(ClassLoaderData* cld) {
930 cld->is_alive();
931 }
932 };
933
934 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
935 public:
936 void do_nmethod(nmethod* n) {
937 n->is_unloading();
938 }
939 };
940
941 // This task not only evacuates/updates marked weak roots, but also "null"
942 // dead weak roots.
943 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
944 private:
945 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
946
947 // Roots related to concurrent class unloading
948 ShenandoahClassLoaderDataRoots<true /* concurrent */>
949 _cld_roots;
950 ShenandoahConcurrentNMethodIterator _nmethod_itr;
951 ShenandoahGeneration* _generation;
952 ShenandoahPhaseTimings::Phase _phase;
953
954 public:
955 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
956 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
957 _vm_roots(phase),
958 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
959 _nmethod_itr(ShenandoahCodeRoots::table()),
960 _generation(generation),
961 _phase(phase) {}
962
963 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
964 // Notify runtime data structures of potentially dead oops
965 _vm_roots.report_num_dead();
966 }
967
968 void work(uint worker_id) override {
969 ShenandoahConcurrentWorkerSession worker_session(worker_id);
970 SuspendibleThreadSetJoiner sts_join;
971 {
972 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
973 // may race against OopStorage::release() calls.
974 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
975 _vm_roots.oops_do(&cl, worker_id);
976 }
977
978 // If we are going to perform concurrent class unloading later on, we need to
979 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
980 // can clean up immediate garbage sooner.
981 if (ShenandoahHeap::heap()->unload_classes()) {
982 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
983 // CLD's holder or evacuate it.
984 {
985 ShenandoahIsCLDAliveClosure is_cld_alive;
986 _cld_roots.cld_do(&is_cld_alive, worker_id);
987 }
988
989 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
990 // The closure calls nmethod->is_unloading(). The is_unloading
991 // state is cached, therefore, during concurrent class unloading phase,
992 // we will not touch the metadata of unloading nmethods
993 {
994 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
995 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
996 _nmethod_itr.nmethods_do(&is_nmethod_alive);
997 }
998 }
999 }
1000 };
1001
1002 void ShenandoahConcurrentGC::op_weak_roots() {
1003 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1004 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1005 {
1006 // Concurrent weak root processing
1007 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1008 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1009 heap->workers()->run_task(&task);
1010 }
1011
1012 {
1013 // It is possible for mutators executing the load reference barrier to have
1014 // loaded an oop through a weak handle that has since been nulled out by
1015 // weak root processing. Handshaking here forces them to complete the
1016 // barrier before the GC cycle continues and does something that would
1017 // change the evaluation of the barrier (for example, resetting the TAMS
1018 // on trashed regions could make an oop appear to be marked _after_ the
1019 // region has been recycled).
1020 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1021 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1022 }
1023 }
1024
1025 void ShenandoahConcurrentGC::op_class_unloading() {
1026 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1027 assert (heap->is_concurrent_weak_root_in_progress() &&
1028 heap->unload_classes(),
1029 "Checked by caller");
1030 heap->do_class_unloading();
1031 }
1032
1033 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1034 private:
1035 ShenandoahEvacuateUpdateMetadataClosure _cl;
1036
1037 public:
1038 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1039
1040 void do_nmethod(nmethod* n) {
1041 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1042 ShenandoahNMethodLocker locker(data->lock());
1043 data->oops_do(&_cl, /* fix_relocations = */ true);
1044 ShenandoahNMethod::disarm_nmethod(n);
1045 }
1046 };
1047
1048 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1049 private:
1050 ShenandoahPhaseTimings::Phase _phase;
1051 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1052 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1053 _cld_roots;
1054 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1055
1056 public:
1057 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1058 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1059 _phase(phase),
1060 _vm_roots(phase),
1061 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1062 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1063
1064 void work(uint worker_id) {
1065 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1066 {
1067 {
1068 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1069 // may race against OopStorage::release() calls.
1070 ShenandoahContextEvacuateUpdateRootsClosure cl;
1071 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1072 }
1073
1074 {
1075 ShenandoahEvacuateUpdateMetadataClosure cl;
1076 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1077 _cld_roots.cld_do(&clds, worker_id);
1078 }
1079 }
1080
1081 if (!ShenandoahHeap::heap()->unload_classes()) {
1082 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1083 ShenandoahEvacUpdateCodeCacheClosure cl;
1084 _nmethod_itr.nmethods_do(&cl);
1085 }
1086 }
1087 };
1088
1089 void ShenandoahConcurrentGC::op_strong_roots() {
1090 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1091 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1092 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1093 heap->workers()->run_task(&task);
1094 heap->set_concurrent_strong_root_in_progress(false);
1095 }
1096
1097 void ShenandoahConcurrentGC::op_cleanup_early() {
1098 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1099 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1100 "cleanup early.");
1101 ShenandoahHeap::heap()->recycle_trash();
1102 }
1103
1104 void ShenandoahConcurrentGC::op_evacuate() {
1105 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1106 }
1107
1108 void ShenandoahConcurrentGC::op_init_update_refs() {
1109 if (ShenandoahVerify) {
1110 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1111 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1112 heap->verifier()->verify_before_update_refs(_generation);
1113 }
1114 }
1115
1116 void ShenandoahConcurrentGC::op_update_refs() {
1117 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1118 }
1119
1120 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1121 private:
1122 // This closure runs when thread is stopped for handshake, which means
1123 // we can use non-concurrent closure here, as long as it only updates
1124 // locations modified by the thread itself, i.e. stack locations.
1125 ShenandoahNonConcUpdateRefsClosure _cl;
1126 public:
1127 ShenandoahUpdateThreadHandshakeClosure();
1128 void do_thread(Thread* thread) override;
1129 };
1130
1131 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1132 HandshakeClosure("Shenandoah Update Thread Roots") {
1133 }
1134
1135 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1136 if (thread->is_Java_thread()) {
1137 JavaThread* jt = JavaThread::cast(thread);
1138 ResourceMark rm;
1139 jt->oops_do(&_cl, nullptr);
1140 }
1141 }
1142
1143 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1144 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1145 // to pick up overwritten pointers that point into a cset region. If these pointers
1146 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1147 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1148 //
1149 // Therefore, at the end of update refs, we use this closure to update the thread roots
1150 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1151 // anything that has already been marked or anything that points to a region which is
1152 // not old. We do not need to worry about ABA situations where a region may become old
1153 // after the pointer is enqueued but before it is filtered. There are only two ways a
1154 // region may become old:
1155 // 1. The region is promoted in place. This is safe because such regions will never
1156 // be in the collection set. If this happens, the pointer will be preserved, essentially
1157 // becoming part of the old snapshot.
1158 // 2. The region is allocated during evacuation of old. This is also not a concern because
1159 // we haven't yet finished marking old so no mixed evacuations will happen.
1160 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1161 ShenandoahFlushSATB _flush_all_satb;
1162
1163 public:
1164 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1165 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1166 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1167 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1168 "Should be filtering pointers outside of old during old marking");
1169 }
1170
1171 void do_thread(Thread* thread) override {
1172 _update_roots.do_thread(thread);
1173 _flush_all_satb.do_thread(thread);
1174 }
1175 };
1176
1177 void ShenandoahConcurrentGC::op_update_thread_roots() {
1178 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1179 if (heap->is_concurrent_old_mark_in_progress()) {
1180 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1181 Handshake::execute(&cl);
1182 } else {
1183 ShenandoahUpdateThreadHandshakeClosure cl;
1184 Handshake::execute(&cl);
1185 }
1186 }
1187
1188 void ShenandoahConcurrentGC::op_final_update_refs() {
1189 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1190 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1191 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1192
1193 heap->finish_concurrent_roots();
1194
1195 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1196 // everything.
1197 if (heap->cancelled_gc()) {
1198 heap->clear_cancelled_gc();
1199 }
1200
1201 // Has to be done before cset is clear
1202 if (ShenandoahVerify) {
1203 heap->verifier()->verify_roots_in_to_space(_generation);
1204 }
1205
1206 // If we are running in generational mode and this is an aging cycle, this will also age active
1207 // regions that haven't been used for allocation.
1208 heap->update_heap_region_states(true /*concurrent*/);
1209
1210 heap->set_update_refs_in_progress(false);
1211 heap->set_has_forwarded_objects(false);
1212
1213 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1214 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1215 // entire regions. Both of these relevant operations occur before final update refs.
1216 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1217 }
1218
1219 if (ShenandoahVerify) {
1220 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1221 heap->verifier()->verify_after_update_refs(_generation);
1222 }
1223
1224 if (VerifyAfterGC) {
1225 Universe::verify();
1226 }
1227
1228 heap->rebuild_free_set(true /*concurrent*/);
1229 _generation->heuristics()->start_idle_span();
1230
1231 {
1232 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1233 heap->propagate_gc_state_to_all_threads();
1234 }
1235 }
1236
1237 bool ShenandoahConcurrentGC::entry_final_roots() {
1238 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1239 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1240
1241
1242 const char* msg = conc_final_roots_event_message();
1243 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1244 EventMark em("%s", msg);
1245 ShenandoahWorkerScope scope(heap->workers(),
1246 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
1247 msg);
1248
1249 if (heap->mode()->is_generational()) {
1250 if (!complete_abbreviated_cycle()) {
1251 return false;
1252 }
1253 }
1254
1255 heap->concurrent_final_roots();
1256 return true;
1257 }
1258
1259 void ShenandoahConcurrentGC::op_verify_final() {
1260 assert(ShenandoahVerify, "Should have been checked before");
1261 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1262 heap->verifier()->verify_after_gc(_generation);
1263 }
1264
1265 void ShenandoahConcurrentGC::op_cleanup_complete() {
1266 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1267 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1268 "cleanup complete.");
1269 ShenandoahHeap::heap()->recycle_trash();
1270 }
1271
1272 void ShenandoahConcurrentGC::op_reset_after_collect() {
1273 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1274 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1275 "reset after collection.");
1276
1277 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1278 if (heap->mode()->is_generational()) {
1279 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1280 // the young generation intact. In particular, reference processing in the old generation may potentially
1281 // need the reachability of a young generation referent of a Reference object in the old generation.
1282 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1283 heap->young_generation()->reset_mark_bitmap<false>();
1284 }
1285 } else {
1286 _generation->reset_mark_bitmap<false>();
1287 }
1288 }
1289
1290 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1291 if (ShenandoahHeap::heap()->cancelled_gc()) {
1292 _degen_point = point;
1293 return true;
1294 }
1295 return false;
1296 }
1297
1298 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1299 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1300 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1301 if (heap->unload_classes()) {
1302 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1303 } else {
1304 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1305 }
1306 }
1307
1308 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1309 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1310 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1311 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1312
1313 if (heap->unload_classes()) {
1314 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1315 } else {
1316 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1317 }
1318 }
1319
1320 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1321 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1322 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1323 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1324 if (heap->unload_classes()) {
1325 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1326 } else {
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1328 }
1329 }
1330
1331 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1332 if (ShenandoahHeap::heap()->unload_classes()) {
1333 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1334 } else {
1335 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1336 }
1337 }
1338
1339 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1340 if (ShenandoahHeap::heap()->unload_classes()) {
1341 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1342 } else {
1343 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1344 }
1345 }
1346
1347 const char* ShenandoahConcurrentGC::verify_final_event_message() const {
1348 if (ShenandoahHeap::heap()->unload_classes()) {
1349 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)");
1350 } else {
1351 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
1352 }
1353 }
1354
1355 const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
1356 if (ShenandoahHeap::heap()->unload_classes()) {
1357 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
1358 } else {
1359 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
1360 }
1361 }
1362
1363 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1364 if (ShenandoahHeap::heap()->unload_classes()) {
1365 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1366 } else {
1367 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1368 }
1369 }
1370
1371 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1372 if (ShenandoahHeap::heap()->unload_classes()) {
1373 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1374 } else {
1375 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1376 }
1377 }
1378
1379 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1380 if (ShenandoahHeap::heap()->unload_classes()) {
1381 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1382 } else {
1383 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1384 }
1385 }
1386
1387 const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
1388 if (ShenandoahHeap::heap()->unload_classes()) {
1389 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
1390 } else {
1391 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
1392 }
1393 }