1 /*
2 * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectorCounters.hpp"
30 #include "gc/shared/continuationGCSupport.inline.hpp"
31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
32 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
35 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
37 #include "gc/shenandoah/shenandoahGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
39 #include "gc/shenandoah/shenandoahLock.hpp"
40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
42 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
52 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
53 #include "memory/allocation.hpp"
54 #include "prims/jvmtiTagMap.hpp"
55 #include "runtime/vmThread.hpp"
56 #include "utilities/events.hpp"
57
58 // Breakpoint support
59 class ShenandoahBreakpointGCScope : public StackObj {
60 private:
61 const GCCause::Cause _cause;
62 public:
63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
64 if (cause == GCCause::_wb_breakpoint) {
65 ShenandoahBreakpoint::start_gc();
66 ShenandoahBreakpoint::at_before_gc();
67 }
68 }
69
70 ~ShenandoahBreakpointGCScope() {
71 if (_cause == GCCause::_wb_breakpoint) {
72 ShenandoahBreakpoint::at_after_gc();
73 }
74 }
75 };
76
77 class ShenandoahBreakpointMarkScope : public StackObj {
78 private:
79 const GCCause::Cause _cause;
80 public:
81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
82 if (_cause == GCCause::_wb_breakpoint) {
83 ShenandoahBreakpoint::at_after_marking_started();
84 }
85 }
86
87 ~ShenandoahBreakpointMarkScope() {
88 if (_cause == GCCause::_wb_breakpoint) {
89 ShenandoahBreakpoint::at_before_marking_completed();
90 }
91 }
92 };
93
94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
95 ShenandoahGC(generation),
96 _mark(generation),
97 _degen_point(ShenandoahDegenPoint::_degenerated_unset),
98 _abbreviated(false),
99 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
100 }
101
102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
103 return _degen_point;
104 }
105
106 void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
107 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
108 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent init update refs", "");
109 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
110 EventMark em("%s", msg);
111
112 heap->try_inject_pin();
113 // Evacuation is complete, retire gc labs and change gc state
114 heap->concurrent_prepare_for_update_refs();
115 }
116
117 void ShenandoahConcurrentGC::entry_update_card_table() {
118 ShenandoahHeap* const heap = ShenandoahHeap::heap();
119 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
120 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update cards", "");
121 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
122 EventMark em("%s", msg);
123
124 ShenandoahWorkerScope scope(heap->workers(),
125 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
126 "concurrent update cards");
127
128 heap->try_inject_pin();
129 // Heap needs to be parsable here.
130 // Also, parallel heap region iterate must have a phase set.
131 assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
132 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
133 }
134
135 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
136 ShenandoahHeap* const heap = ShenandoahHeap::heap();
137 _generation->ref_processor()->set_soft_reference_policy(
138 GCCause::should_clear_all_soft_refs(cause));
139
140 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
141
142 // Reset for upcoming marking
143 entry_reset();
144
145 // Start initial mark under STW
146 vmop_entry_init_mark();
147
148 {
149 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
150
151 // Reset task queue stats here, rather than in mark_concurrent_roots,
152 // because remembered set scan will `push` oops into the queues and
153 // resetting after this happens will lose those counts.
154 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
155
156 // Concurrent remembered set scanning
157 entry_scan_remembered_set();
158
159 // Concurrent mark roots
160 entry_mark_roots();
161 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
162 return false;
163 }
164
165 // Continue concurrent mark
166 entry_mark();
167 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
168 return false;
169 }
170 }
171
172 // Complete marking under STW, and start evacuation
173 vmop_entry_final_mark();
174
175 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still
176 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
177 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
178 // from that phase.
179 if (_generation->is_concurrent_mark_in_progress()) {
180 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
181 assert(cancelled, "GC must have been cancelled between concurrent and final mark");
182 return false;
183 }
184
185 assert(heap->is_concurrent_weak_root_in_progress(), "Must be doing weak roots now");
186
187 // Finish all thread/stack roots if needed. This completes stack watermark processing.
188 if (heap->is_evacuation_in_progress()) {
189 entry_thread_roots();
190 }
191
192 // Process weak roots that might still point to regions that would be broken by cleanup.
193 // We cannot recycle regions because weak roots need to know what is marked in trashed regions.
194 entry_weak_refs();
195 entry_weak_roots();
196
197 // Perform concurrent class unloading before any regions get recycled. Class unloading may
198 // need to inspect unmarked objects in trashed regions.
199 if (heap->unload_classes()) {
200 entry_class_unloading();
201 }
202
203 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
204 // the space. This would be the last action if there is nothing to evacuate. Note that
205 // we will not age young-gen objects in the case that we skip evacuation.
206 entry_cleanup_early();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Roots processing is complete, put the weak roots flag down.
216 vmop_entry_final_roots();
217
218 // Continue the cycle with evacuation and optional update-refs.
219 // This may be skipped if there is nothing to evacuate.
220 // If so, evac_in_progress would be unset by collection set preparation code.
221 if (heap->is_evacuation_in_progress()) {
222 // Concurrently evacuate
223 entry_evacuate();
224 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
225 return false;
226 }
227
228 // Perform update-refs phase.
229 entry_concurrent_update_refs_prepare(heap);
230
231 if (ShenandoahHeap::heap()->mode()->is_generational()) {
232 entry_update_card_table();
233 }
234
235 if (ShenandoahVerify) {
236 vmop_entry_init_update_refs();
237 }
238
239 entry_update_refs();
240 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
241 return false;
242 }
243
244 // Concurrent update thread roots
245 entry_update_thread_roots();
246 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
247 return false;
248 }
249
250 vmop_entry_final_update_refs();
251
252 // Update references freed up collection set, kick the cleanup to reclaim the space.
253 entry_cleanup_complete();
254 } else {
255 _abbreviated = true;
256
257 if (heap->mode()->is_generational()) {
258 entry_complete_abbreviated_cycle();
259
260 // If the promote-in-place operation was cancelled, we can have the degenerated
261 // cycle complete the operation. It will see that no evacuations are in progress,
262 // and that there are regions wanting promotion. The risk with not handling the
263 // cancellation would be failing to restore top for these regions and leaving
264 // them unable to serve allocations for the old generation.
265 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
266 return false;
267 }
268 }
269
270 // In normal cycle, final-update-refs would verify at the end of the cycle.
271 // In abbreviated cycle, we need to verify separately.
272 if (ShenandoahVerify) {
273 vmop_entry_final_verify();
274 }
275 }
276
277 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
278 // abbreviated cycle.
279 if (heap->mode()->is_generational()) {
280 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
281 }
282
283 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
284 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
285 // reducing the likelihood that GC will degenerate.
286 entry_reset_after_collect();
287
288 return true;
289 }
290
291 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
292 shenandoah_assert_generational();
293
294 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
295
296 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
297 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent complete abbreviated cycle", "");
298 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
299 EventMark em("%s", msg);
300
301 ShenandoahWorkerScope scope(heap->workers(),
302 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
303 msg);
304
305 heap->try_inject_pin();
306 // We chose not to evacuate because we found sufficient immediate garbage.
307 // However, there may still be regions to promote in place, so do that now.
308 if (heap->old_generation()->has_in_place_promotions()) {
309 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
310 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
311 heap->promote_regions_in_place(_generation, true);
312 }
313
314 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
315 // the control thread will detect it on its next iteration and run a degenerated young cycle.
316 if (!heap->cancelled_gc() && !_generation->is_old()) {
317 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
318 heap->update_region_ages(_generation->complete_marking_context());
319 }
320 }
321
322 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
323 ShenandoahHeap* const heap = ShenandoahHeap::heap();
324 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
325 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
326
327 heap->try_inject_alloc_failure();
328 VM_ShenandoahInitMark op(this);
329 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
330 }
331
332 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
333 ShenandoahHeap* const heap = ShenandoahHeap::heap();
334 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
335 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
336
337 heap->try_inject_alloc_failure();
338 VM_ShenandoahFinalMarkStartEvac op(this);
339 VMThread::execute(&op); // jump to entry_final_mark under safepoint
340 heap->try_inject_pin();
341 }
342
343 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
344 ShenandoahHeap* const heap = ShenandoahHeap::heap();
345 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
346 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
347
348 heap->try_inject_alloc_failure();
349 heap->try_inject_pin();
350 VM_ShenandoahInitUpdateRefs op(this);
351 VMThread::execute(&op);
352 }
353
354 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
355 ShenandoahHeap* const heap = ShenandoahHeap::heap();
356 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
357 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
358
359 heap->try_inject_alloc_failure();
360 heap->try_inject_pin();
361 VM_ShenandoahFinalUpdateRefs op(this);
362 VMThread::execute(&op);
363 }
364
365 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
366 ShenandoahHeap* const heap = ShenandoahHeap::heap();
367 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
368 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
369
370 // This phase does not use workers, no need for setup
371 heap->try_inject_pin();
372 VM_ShenandoahFinalRoots op(this);
373 VMThread::execute(&op);
374 }
375
376 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
377 ShenandoahHeap* const heap = ShenandoahHeap::heap();
378 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
379 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
380
381 // This phase does not use workers, no need for setup
382 heap->try_inject_alloc_failure();
383 heap->try_inject_pin();
384 VM_ShenandoahFinalVerify op(this);
385 VMThread::execute(&op);
386 }
387
388 void ShenandoahConcurrentGC::entry_init_mark() {
389 ShenandoahHeap* const heap = ShenandoahHeap::heap();
390 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
391
392 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Mark", "");
393 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
394 EventMark em("%s", msg);
395
396 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
397 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
398 "init marking");
399
400 op_init_mark();
401 }
402
403 void ShenandoahConcurrentGC::entry_final_mark() {
404 ShenandoahHeap* const heap = ShenandoahHeap::heap();
405 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
406 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
407
408 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Mark", "");
409 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
410 EventMark em("%s", msg);
411
412 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
413 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
414 "final marking");
415
416 op_final_mark();
417 }
418
419 void ShenandoahConcurrentGC::entry_init_update_refs() {
420 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Update Refs", "");
421 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
422 EventMark em("%s", msg);
423
424 // No workers used in this phase, no setup required
425 op_init_update_refs();
426 }
427
428 void ShenandoahConcurrentGC::entry_final_update_refs() {
429 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Update Refs", "");
430 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
431 EventMark em("%s", msg);
432
433 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
434 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
435 "final reference update");
436
437 op_final_update_refs();
438 }
439
440 void ShenandoahConcurrentGC::entry_final_verify() {
441 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Verify Final", "");
442 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
443 EventMark em("%s", msg);
444
445 op_verify_final();
446 }
447
448 void ShenandoahConcurrentGC::entry_reset() {
449 ShenandoahHeap* const heap = ShenandoahHeap::heap();
450 heap->release_injected_pins();
451 heap->try_inject_alloc_failure();
452
453 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
454 {
455 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset", "");
456 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
457 EventMark em("%s", msg);
458
459 ShenandoahWorkerScope scope(heap->workers(),
460 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
461 msg);
462 op_reset();
463 }
464 }
465
466 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
467 if (_generation->is_young()) {
468 ShenandoahHeap* const heap = ShenandoahHeap::heap();
469 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
470
471 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent remembered set scanning", "");
472 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
473 EventMark em("%s", msg);
474
475 ShenandoahWorkerScope scope(heap->workers(),
476 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
477 msg);
478
479 heap->try_inject_alloc_failure();
480 _generation->scan_remembered_set(true /* is_concurrent */);
481 }
482 }
483
484 void ShenandoahConcurrentGC::entry_mark_roots() {
485 ShenandoahHeap* const heap = ShenandoahHeap::heap();
486 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
487 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking roots", "");
488 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
489 EventMark em("%s", msg);
490
491 ShenandoahWorkerScope scope(heap->workers(),
492 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
493 "concurrent marking roots");
494
495 heap->try_inject_alloc_failure();
496 op_mark_roots();
497 }
498
499 void ShenandoahConcurrentGC::entry_mark() {
500 ShenandoahHeap* const heap = ShenandoahHeap::heap();
501 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
502 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
503
504 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
505 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent marking", "");
506 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
507 EventMark em("%s", msg);
508
509 ShenandoahWorkerScope scope(heap->workers(),
510 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
511 "concurrent marking");
512
513 heap->try_inject_alloc_failure();
514 op_mark();
515 heap->try_inject_pin();
516 }
517
518 void ShenandoahConcurrentGC::entry_thread_roots() {
519 ShenandoahHeap* const heap = ShenandoahHeap::heap();
520 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent thread roots", "");
521 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
522 EventMark em("%s", msg);
523
524 ShenandoahWorkerScope scope(heap->workers(),
525 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
526 msg);
527
528 heap->try_inject_alloc_failure();
529 heap->try_inject_pin();
530 op_thread_roots();
531 }
532
533 void ShenandoahConcurrentGC::entry_weak_refs() {
534 ShenandoahHeap* const heap = ShenandoahHeap::heap();
535 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak references", "");
536 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
537 EventMark em("%s", msg);
538
539 ShenandoahWorkerScope scope(heap->workers(),
540 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
541 "concurrent weak references");
542
543 heap->try_inject_alloc_failure();
544 heap->try_inject_pin();
545 op_weak_refs();
546 }
547
548 void ShenandoahConcurrentGC::entry_weak_roots() {
549 ShenandoahHeap* const heap = ShenandoahHeap::heap();
550 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
551 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent weak roots", "");
552 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
553 EventMark em("%s", msg);
554
555 ShenandoahWorkerScope scope(heap->workers(),
556 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
557 "concurrent weak root");
558
559 heap->try_inject_alloc_failure();
560 heap->try_inject_pin();
561 op_weak_roots();
562 }
563
564 void ShenandoahConcurrentGC::entry_class_unloading() {
565 ShenandoahHeap* const heap = ShenandoahHeap::heap();
566 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
567 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent class unloading", "");
568 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
569 EventMark em("%s", msg);
570
571 ShenandoahWorkerScope scope(heap->workers(),
572 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
573 "concurrent class unloading");
574
575 heap->try_inject_alloc_failure();
576 heap->try_inject_pin();
577 op_class_unloading();
578 }
579
580 void ShenandoahConcurrentGC::entry_strong_roots() {
581 ShenandoahHeap* const heap = ShenandoahHeap::heap();
582 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
583 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent strong roots", "");
584 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
585 EventMark em("%s", msg);
586
587 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
588
589 ShenandoahWorkerScope scope(heap->workers(),
590 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
591 "concurrent strong root");
592
593 heap->try_inject_alloc_failure();
594 heap->try_inject_pin();
595 op_strong_roots();
596 }
597
598 void ShenandoahConcurrentGC::entry_cleanup_early() {
599 ShenandoahHeap* const heap = ShenandoahHeap::heap();
600 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
601 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
602 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
603 EventMark em("%s", msg);
604
605 // This phase does not use workers, no need for setup
606 heap->try_inject_alloc_failure();
607 heap->try_inject_pin();
608 op_cleanup_early();
609 if (!heap->is_evacuation_in_progress()) {
610 // This is an abbreviated cycle. Rebuild the freeset in order to establish reserves for the next GC cycle. Doing
611 // the rebuild ASAP also expedites availability of immediate trash, reducing the likelihood that we will degenerate
612 // during promote-in-place processing.
613 heap->rebuild_free_set(true /*concurrent*/);
614 }
615 }
616
617 void ShenandoahConcurrentGC::entry_evacuate() {
618 ShenandoahHeap* const heap = ShenandoahHeap::heap();
619 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
620 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent evacuation", "");
621 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
622 EventMark em("%s", msg);
623
624 ShenandoahWorkerScope scope(heap->workers(),
625 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
626 "concurrent evacuation");
627
628 heap->try_inject_alloc_failure();
629 heap->try_inject_pin();
630 op_evacuate();
631 }
632
633 void ShenandoahConcurrentGC::entry_update_thread_roots() {
634 ShenandoahHeap* const heap = ShenandoahHeap::heap();
635 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
636 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update thread roots", "");
637 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
638 EventMark em("%s", msg);
639
640 // No workers used in this phase, no setup required
641 heap->try_inject_alloc_failure();
642 heap->try_inject_pin();
643 op_update_thread_roots();
644 }
645
646 void ShenandoahConcurrentGC::entry_update_refs() {
647 ShenandoahHeap* const heap = ShenandoahHeap::heap();
648 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
649 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent update references", "");
650 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
651 EventMark em("%s", msg);
652
653 ShenandoahWorkerScope scope(heap->workers(),
654 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
655 "concurrent reference update");
656
657 heap->try_inject_alloc_failure();
658 heap->try_inject_pin();
659 op_update_refs();
660 }
661
662 void ShenandoahConcurrentGC::entry_cleanup_complete() {
663 ShenandoahHeap* const heap = ShenandoahHeap::heap();
664 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
665 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent cleanup", "");
666 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
667 EventMark em("%s", msg);
668
669 // This phase does not use workers, no need for setup
670 heap->try_inject_alloc_failure();
671 op_cleanup_complete();
672 }
673
674 void ShenandoahConcurrentGC::entry_reset_after_collect() {
675 ShenandoahHeap* const heap = ShenandoahHeap::heap();
676 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
677 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent reset after collect", "");
678 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_after_collect);
679 EventMark em("%s", msg);
680
681 op_reset_after_collect();
682 }
683
684 void ShenandoahConcurrentGC::op_reset() {
685 ShenandoahHeap* const heap = ShenandoahHeap::heap();
686
687 // If it is old GC bootstrap cycle, always clear bitmap for global gen
688 // to ensure bitmap for old gen is clear for old GC cycle after this.
689 if (_do_old_gc_bootstrap) {
690 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot reset old without making it parsable");
691 heap->global_generation()->prepare_gc();
692 } else {
693 _generation->prepare_gc();
694 }
695
696 if (heap->mode()->is_generational()) {
697 heap->old_generation()->card_scan()->mark_read_table_as_clean();
698 }
699 }
700
701 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
702 private:
703 ShenandoahMarkingContext* const _ctx;
704 public:
705 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
706
707 void heap_region_do(ShenandoahHeapRegion* r) {
708 assert(!r->has_live(), "Region %zu should have no live data", r->index());
709 if (r->is_active()) {
710 // Check if region needs updating its TAMS. We have updated it already during concurrent
711 // reset, so it is very likely we don't need to do another write here. Since most regions
712 // are not "active", this path is relatively rare.
713 if (_ctx->top_at_mark_start(r) != r->top()) {
714 _ctx->capture_top_at_mark_start(r);
715 }
716 } else {
717 assert(_ctx->top_at_mark_start(r) == r->top(),
718 "Region %zu should already have correct TAMS", r->index());
719 }
720 }
721
722 bool is_thread_safe() { return true; }
723 };
724
725 void ShenandoahConcurrentGC::start_mark() {
726 _mark.start_mark();
727 }
728
729 void ShenandoahConcurrentGC::op_init_mark() {
730 ShenandoahHeap* const heap = ShenandoahHeap::heap();
731 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
732 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
733
734 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
735 assert(!_generation->is_mark_complete(), "should not be complete");
736 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
737
738 if (heap->mode()->is_generational()) {
739 if (_generation->is_global()) {
740 heap->old_generation()->cancel_gc();
741 }
742
743 {
744 // After we swap card table below, the write-table is all clean, and the read table holds
745 // cards dirty prior to the start of GC. Young and bootstrap collection will update
746 // the write card table as a side effect of remembered set scanning. Global collection will
747 // update the card table as a side effect of global marking of old objects.
748 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
749 _generation->swap_card_tables();
750 }
751 }
752
753 if (ShenandoahVerify) {
754 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
755 heap->verifier()->verify_before_concmark(_generation);
756 }
757
758 if (VerifyBeforeGC) {
759 Universe::verify();
760 }
761
762 _generation->set_concurrent_mark_in_progress(true);
763
764 start_mark();
765
766 if (_do_old_gc_bootstrap) {
767 shenandoah_assert_generational();
768 // Update region state for both young and old regions
769 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
770 ShenandoahInitMarkUpdateRegionStateClosure cl;
771 heap->parallel_heap_region_iterate(&cl);
772 heap->old_generation()->ref_processor()->reset_thread_locals();
773 } else {
774 // Update region state for only young regions
775 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
776 ShenandoahInitMarkUpdateRegionStateClosure cl;
777 _generation->parallel_heap_region_iterate(&cl);
778 }
779
780 // Weak reference processing
781 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
782 rp->reset_thread_locals();
783
784 // Make above changes visible to worker threads
785 OrderAccess::fence();
786
787 // Arm nmethods/stack for concurrent processing
788 CodeCache::arm_all_nmethods();
789
790 {
791 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
792 heap->propagate_gc_state_to_all_threads();
793 }
794 }
795
796 void ShenandoahConcurrentGC::op_mark_roots() {
797 _mark.mark_concurrent_roots();
798 }
799
800 void ShenandoahConcurrentGC::op_mark() {
801 _mark.concurrent_mark();
802 }
803
804 void ShenandoahConcurrentGC::op_final_mark() {
805 ShenandoahHeap* const heap = ShenandoahHeap::heap();
806 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
807 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
808
809 if (ShenandoahVerify) {
810 heap->verifier()->verify_roots_no_forwarded(_generation);
811 }
812
813 if (!heap->cancelled_gc()) {
814 _mark.finish_mark();
815 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
816
817 // Notify JVMTI that the tagmap table will need cleaning.
818 JvmtiTagMap::set_needs_cleaning();
819
820 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
821 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
822 // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
823 _generation->prepare_regions_and_collection_set(true /*concurrent*/);
824
825 // Has to be done after cset selection
826 heap->prepare_concurrent_roots();
827
828 if (!heap->collection_set()->is_empty()) {
829 LogTarget(Debug, gc, cset) lt;
830 if (lt.is_enabled()) {
831 ResourceMark rm;
832 LogStream ls(lt);
833 heap->collection_set()->print_on(&ls);
834 }
835
836 if (ShenandoahVerify) {
837 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
838 heap->verifier()->verify_before_evacuation(_generation);
839 }
840
841 heap->set_evacuation_in_progress(true);
842 // From here on, we need to update references.
843 heap->set_has_forwarded_objects(true);
844
845 } else {
846 if (ShenandoahVerify) {
847 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
848 if (has_in_place_promotions(heap)) {
849 heap->verifier()->verify_after_concmark_with_promotions(_generation);
850 } else {
851 heap->verifier()->verify_after_concmark(_generation);
852 }
853 }
854 }
855 }
856
857 // Arm nmethods/stack for concurrent processing
858 CodeCache::arm_all_nmethods();
859
860 {
861 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
862 heap->propagate_gc_state_to_all_threads();
863 }
864 }
865
866 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
867 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
868 }
869
870 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
871 private:
872 OopClosure* const _oops;
873 public:
874 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
875
876 void do_thread(Thread* thread) override {
877 JavaThread* const jt = JavaThread::cast(thread);
878 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
879 }
880 };
881
882 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
883 private:
884 ShenandoahJavaThreadsIterator _java_threads;
885
886 public:
887 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
888 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
889 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
890 }
891
892 void work(uint worker_id) override {
893 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
894 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
895 _java_threads.threads_do(&thr_cl, worker_id);
896 }
897 };
898
899 void ShenandoahConcurrentGC::op_thread_roots() {
900 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
901 assert(heap->is_evacuation_in_progress(), "Checked by caller");
902 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
903 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
904 heap->workers()->run_task(&task);
905 }
906
907 void ShenandoahConcurrentGC::op_weak_refs() {
908 ShenandoahHeap* const heap = ShenandoahHeap::heap();
909 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
910 // Concurrent weak refs processing
911 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
912 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
913 ShenandoahBreakpoint::at_after_reference_processing_started();
914 }
915 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
916 }
917
918 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
919 private:
920 ShenandoahHeap* const _heap;
921 ShenandoahGeneration* const _generation;
922 ShenandoahMarkingContext* const _mark_context;
923 bool _evac_in_progress;
924 Thread* const _thread;
925
926 public:
927 explicit ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation);
928 void do_oop(oop* p);
929 void do_oop(narrowOop* p);
930 };
931
932 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure(ShenandoahGeneration* generation) :
933 _heap(ShenandoahHeap::heap()),
934 _generation(generation),
935 _mark_context(ShenandoahHeap::heap()->marking_context()),
936 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
937 _thread(Thread::current()) {
938 }
939
940 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
941 const oop obj = RawAccess<>::oop_load(p);
942 if (!CompressedOops::is_null(obj)) {
943 if (!_mark_context->is_marked(obj)) {
944 if (_generation->contains(obj)) {
945 // Note: The obj is dead here. Do not touch it, just clear.
946 ShenandoahHeap::atomic_clear_oop(p, obj);
947 }
948 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
949 oop resolved = ShenandoahForwarding::get_forwardee(obj);
950 if (resolved == obj) {
951 resolved = _heap->evacuate_object(obj, _thread);
952 }
953 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
954 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
955 }
956 }
957 }
958
959 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
960 ShouldNotReachHere();
961 }
962
963 class ShenandoahIsCLDAliveClosure : public CLDClosure {
964 public:
965 void do_cld(ClassLoaderData* cld) {
966 cld->is_alive();
967 }
968 };
969
970 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
971 public:
972 void do_nmethod(nmethod* n) {
973 n->is_unloading();
974 }
975 };
976
977 // This task not only evacuates/updates marked weak roots, but also "null"
978 // dead weak roots.
979 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
980 private:
981 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
982
983 // Roots related to concurrent class unloading
984 ShenandoahClassLoaderDataRoots<true /* concurrent */>
985 _cld_roots;
986 ShenandoahConcurrentNMethodIterator _nmethod_itr;
987 ShenandoahGeneration* _generation;
988 ShenandoahPhaseTimings::Phase _phase;
989
990 public:
991 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahGeneration* generation, ShenandoahPhaseTimings::Phase phase) :
992 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
993 _vm_roots(phase),
994 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
995 _nmethod_itr(ShenandoahCodeRoots::table()),
996 _generation(generation),
997 _phase(phase) {}
998
999 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1000 // Notify runtime data structures of potentially dead oops
1001 _vm_roots.report_num_dead();
1002 }
1003
1004 void work(uint worker_id) override {
1005 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1006 SuspendibleThreadSetJoiner sts_join;
1007 {
1008 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1009 // may race against OopStorage::release() calls.
1010 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
1011 _vm_roots.oops_do(&cl, worker_id);
1012 }
1013
1014 // If we are going to perform concurrent class unloading later on, we need to
1015 // clean up the weak oops in CLD and determine nmethod's unloading state, so that we
1016 // can clean up immediate garbage sooner.
1017 if (ShenandoahHeap::heap()->unload_classes()) {
1018 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
1019 // CLD's holder or evacuate it.
1020 {
1021 ShenandoahIsCLDAliveClosure is_cld_alive;
1022 _cld_roots.cld_do(&is_cld_alive, worker_id);
1023 }
1024
1025 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1026 // The closure calls nmethod->is_unloading(). The is_unloading
1027 // state is cached, therefore, during concurrent class unloading phase,
1028 // we will not touch the metadata of unloading nmethods
1029 {
1030 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1031 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1032 _nmethod_itr.nmethods_do(&is_nmethod_alive);
1033 }
1034 }
1035 }
1036 };
1037
1038 void ShenandoahConcurrentGC::op_weak_roots() {
1039 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1040 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
1041 {
1042 // Concurrent weak root processing
1043 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
1044 ShenandoahConcurrentWeakRootsEvacUpdateTask task(_generation, ShenandoahPhaseTimings::conc_weak_roots);
1045 heap->workers()->run_task(&task);
1046 }
1047
1048 {
1049 // It is possible for mutators executing the load reference barrier to have
1050 // loaded an oop through a weak handle that has since been nulled out by
1051 // weak root processing. Handshaking here forces them to complete the
1052 // barrier before the GC cycle continues and does something that would
1053 // change the evaluation of the barrier (for example, resetting the TAMS
1054 // on trashed regions could make an oop appear to be marked _after_ the
1055 // region has been recycled).
1056 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1057 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
1058 }
1059 }
1060
1061 void ShenandoahConcurrentGC::op_class_unloading() {
1062 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1063 assert (heap->is_concurrent_weak_root_in_progress() &&
1064 heap->unload_classes(),
1065 "Checked by caller");
1066 heap->do_class_unloading();
1067 }
1068
1069 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1070 private:
1071 ShenandoahEvacuateUpdateMetadataClosure _cl;
1072
1073 public:
1074 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1075
1076 void do_nmethod(nmethod* n) {
1077 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1078 ShenandoahNMethodLocker locker(data->lock());
1079 data->oops_do(&_cl, /* fix_relocations = */ true);
1080 }
1081 };
1082
1083 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1084 private:
1085 ShenandoahPhaseTimings::Phase _phase;
1086 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1087 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1088 _cld_roots;
1089 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1090
1091 public:
1092 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1093 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1094 _phase(phase),
1095 _vm_roots(phase),
1096 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1097 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1098
1099 void work(uint worker_id) {
1100 ShenandoahConcurrentWorkerSession worker_session(worker_id);
1101 {
1102 {
1103 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1104 // may race against OopStorage::release() calls.
1105 ShenandoahContextEvacuateUpdateRootsClosure cl;
1106 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1107 }
1108
1109 {
1110 ShenandoahEvacuateUpdateMetadataClosure cl;
1111 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1112 _cld_roots.cld_do(&clds, worker_id);
1113 }
1114 }
1115
1116 if (!ShenandoahHeap::heap()->unload_classes()) {
1117 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCache, worker_id);
1118 ShenandoahEvacUpdateCodeCacheClosure cl;
1119 _nmethod_itr.nmethods_do(&cl);
1120 }
1121 }
1122 };
1123
1124 void ShenandoahConcurrentGC::op_strong_roots() {
1125 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1126 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1127 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1128 heap->workers()->run_task(&task);
1129 heap->set_concurrent_strong_root_in_progress(false);
1130 }
1131
1132 void ShenandoahConcurrentGC::op_cleanup_early() {
1133 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1134 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1135 "cleanup early.");
1136 ShenandoahHeap::heap()->recycle_trash();
1137 }
1138
1139 void ShenandoahConcurrentGC::op_evacuate() {
1140 ShenandoahHeap::heap()->evacuate_collection_set(_generation, true /*concurrent*/);
1141 }
1142
1143 void ShenandoahConcurrentGC::op_init_update_refs() {
1144 if (ShenandoahVerify) {
1145 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1146 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
1147 heap->verifier()->verify_before_update_refs(_generation);
1148 }
1149 }
1150
1151 void ShenandoahConcurrentGC::op_update_refs() {
1152 ShenandoahHeap::heap()->update_heap_references(_generation, true /*concurrent*/);
1153 }
1154
1155 class ShenandoahUpdateThreadHandshakeClosure : public HandshakeClosure {
1156 private:
1157 // This closure runs when thread is stopped for handshake, which means
1158 // we can use non-concurrent closure here, as long as it only updates
1159 // locations modified by the thread itself, i.e. stack locations.
1160 ShenandoahNonConcUpdateRefsClosure _cl;
1161 public:
1162 ShenandoahUpdateThreadHandshakeClosure();
1163 void do_thread(Thread* thread) override;
1164 };
1165
1166 ShenandoahUpdateThreadHandshakeClosure::ShenandoahUpdateThreadHandshakeClosure() :
1167 HandshakeClosure("Shenandoah Update Thread Roots") {
1168 }
1169
1170 void ShenandoahUpdateThreadHandshakeClosure::do_thread(Thread* thread) {
1171 if (thread->is_Java_thread()) {
1172 JavaThread* jt = JavaThread::cast(thread);
1173 ResourceMark rm;
1174 jt->oops_do(&_cl, nullptr);
1175 }
1176 }
1177
1178 class ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers final : public HandshakeClosure {
1179 // When Shenandoah is marking the old generation, it is possible for the SATB barrier
1180 // to pick up overwritten pointers that point into a cset region. If these pointers
1181 // are accessed by mark threads, they will crash. Once update refs has completed, it is
1182 // no longer possible for a mutator thread to overwrite a pointer into a cset region.
1183 //
1184 // Therefore, at the end of update refs, we use this closure to update the thread roots
1185 // and 'complete' all the thread local SATB buffers. Completing these will filter out
1186 // anything that has already been marked or anything that points to a region which is
1187 // not old. We do not need to worry about ABA situations where a region may become old
1188 // after the pointer is enqueued but before it is filtered. There are only two ways a
1189 // region may become old:
1190 // 1. The region is promoted in place. This is safe because such regions will never
1191 // be in the collection set. If this happens, the pointer will be preserved, essentially
1192 // becoming part of the old snapshot.
1193 // 2. The region is allocated during evacuation of old. This is also not a concern because
1194 // we haven't yet finished marking old so no mixed evacuations will happen.
1195 ShenandoahUpdateThreadHandshakeClosure _update_roots;
1196 ShenandoahFlushSATB _flush_all_satb;
1197
1198 public:
1199 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers() :
1200 HandshakeClosure("Shenandoah Update Thread Roots and Flush SATB"),
1201 _flush_all_satb(ShenandoahBarrierSet::satb_mark_queue_set()) {
1202 assert(ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
1203 "Should be filtering pointers outside of old during old marking");
1204 }
1205
1206 void do_thread(Thread* thread) override {
1207 _update_roots.do_thread(thread);
1208 _flush_all_satb.do_thread(thread);
1209 }
1210 };
1211
1212 void ShenandoahConcurrentGC::op_update_thread_roots() {
1213 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1214 if (heap->is_concurrent_old_mark_in_progress()) {
1215 ShenandoahUpdateThreadRootsAndFlushOldSatbBuffers cl;
1216 Handshake::execute(&cl);
1217 } else {
1218 ShenandoahUpdateThreadHandshakeClosure cl;
1219 Handshake::execute(&cl);
1220 }
1221 }
1222
1223 void ShenandoahConcurrentGC::op_final_update_refs() {
1224 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1225 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1226 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1227
1228 heap->finish_concurrent_roots();
1229
1230 // Clear cancelled GC, if set. On cancellation path, the block before would handle
1231 // everything.
1232 if (heap->cancelled_gc()) {
1233 heap->clear_cancelled_gc();
1234 }
1235
1236 // Has to be done before cset is clear
1237 if (ShenandoahVerify) {
1238 heap->verifier()->verify_roots_in_to_space(_generation);
1239 }
1240
1241 // If we are running in generational mode, this will also age active regions that
1242 // haven't been used for allocation.
1243 heap->update_heap_region_states(true /*concurrent*/);
1244
1245 heap->set_update_refs_in_progress(false);
1246 heap->set_has_forwarded_objects(false);
1247
1248 if (ShenandoahVerify) {
1249 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1250 heap->verifier()->verify_after_update_refs(_generation);
1251 }
1252
1253 if (VerifyAfterGC) {
1254 Universe::verify();
1255 }
1256
1257 heap->rebuild_free_set(true /*concurrent*/);
1258 _generation->heuristics()->start_idle_span();
1259
1260 // Final pause: update GC barriers to idle state.
1261 CodeCache::arm_all_nmethods();
1262
1263 {
1264 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1265 heap->propagate_gc_state_to_all_threads();
1266 }
1267 }
1268
1269 void ShenandoahConcurrentGC::entry_final_roots() {
1270 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1271 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Roots", "");
1272 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1273 EventMark em("%s", msg);
1274
1275 heap->op_final_roots();
1276 }
1277
1278 void ShenandoahConcurrentGC::op_verify_final() {
1279 assert(ShenandoahVerify, "Should have been checked before");
1280 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1281 heap->verifier()->verify_after_gc(_generation);
1282 }
1283
1284 void ShenandoahConcurrentGC::op_cleanup_complete() {
1285 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1286 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1287 "cleanup complete.");
1288 ShenandoahHeap::heap()->recycle_trash();
1289 }
1290
1291 void ShenandoahConcurrentGC::op_reset_after_collect() {
1292 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1293 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1294 "reset after collection.");
1295
1296 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1297 if (heap->mode()->is_generational()) {
1298 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1299 // the young generation intact. In particular, reference processing in the old generation may potentially
1300 // need the reachability of a young generation referent of a Reference object in the old generation.
1301 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1302 heap->young_generation()->reset_mark_bitmap<false>();
1303 }
1304 } else {
1305 _generation->reset_mark_bitmap<false>();
1306 }
1307 }
1308
1309 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1310 if (ShenandoahHeap::heap()->cancelled_gc()) {
1311 _degen_point = point;
1312 return true;
1313 }
1314 return false;
1315 }
--- EOF ---