1 /*
2 * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/barrierSetNMethod.hpp"
28 #include "gc/shared/collectorCounters.hpp"
29 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
30 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
31 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahLock.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
36 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
37 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
38 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
39 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
40 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "gc/shenandoah/shenandoahVerifier.hpp"
43 #include "gc/shenandoah/shenandoahVMOperations.hpp"
44 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
46 #include "memory/allocation.hpp"
47 #include "prims/jvmtiTagMap.hpp"
48 #include "runtime/vmThread.hpp"
49 #include "utilities/events.hpp"
50
51 // Breakpoint support
52 class ShenandoahBreakpointGCScope : public StackObj {
53 public:
54 ShenandoahBreakpointGCScope() {
55 ShenandoahBreakpoint::at_before_gc();
56 }
57
58 ~ShenandoahBreakpointGCScope() {
59 ShenandoahBreakpoint::at_after_gc();
60 }
61 };
62
63 class ShenandoahBreakpointMarkScope : public StackObj {
64 public:
65 ShenandoahBreakpointMarkScope() {
66 ShenandoahBreakpoint::at_after_marking_started();
67 }
68
69 ~ShenandoahBreakpointMarkScope() {
70 ShenandoahBreakpoint::at_before_marking_completed();
71 }
72 };
73
74 ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
75 _mark(),
76 _degen_point(ShenandoahDegenPoint::_degenerated_unset) {
77 }
78
79 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
80 return _degen_point;
81 }
82
83 void ShenandoahConcurrentGC::cancel() {
84 ShenandoahConcurrentMark::cancel();
85 }
86
87 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
88 ShenandoahHeap* const heap = ShenandoahHeap::heap();
89 if (cause == GCCause::_wb_breakpoint) {
90 ShenandoahBreakpoint::start_gc();
91 }
92 ShenandoahBreakpointGCScope breakpoint_gc_scope;
93
94 // Reset for upcoming marking
95 entry_reset();
96
97 // Start initial mark under STW
98 vmop_entry_init_mark();
99
100 {
101 ShenandoahBreakpointMarkScope breakpoint_mark_scope;
102 // Concurrent mark roots
103 entry_mark_roots();
104 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false;
105
106 // Continue concurrent mark
107 entry_mark();
108 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false;
109 }
110
111 // Complete marking under STW, and start evacuation
112 vmop_entry_final_mark();
113
114 // Concurrent stack processing
115 if (heap->is_evacuation_in_progress()) {
116 entry_thread_roots();
117 }
118
119 // Process weak roots that might still point to regions that would be broken by cleanup
120 if (heap->is_concurrent_weak_root_in_progress()) {
121 entry_weak_refs();
122 entry_weak_roots();
123 }
124
125 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
126 // the space. This would be the last action if there is nothing to evacuate.
127 entry_cleanup_early();
128
129 {
130 ShenandoahHeapLocker locker(heap->lock());
131 heap->free_set()->log_status();
132 }
133
134 // Perform concurrent class unloading
135 if (heap->unload_classes() &&
136 heap->is_concurrent_weak_root_in_progress()) {
137 entry_class_unloading();
138 }
139
140 // Processing strong roots
141 // This may be skipped if there is nothing to update/evacuate.
142 // If so, strong_root_in_progress would be unset.
143 if (heap->is_concurrent_strong_root_in_progress()) {
144 entry_strong_roots();
145 }
146
147 // Continue the cycle with evacuation and optional update-refs.
148 // This may be skipped if there is nothing to evacuate.
149 // If so, evac_in_progress would be unset by collection set preparation code.
150 if (heap->is_evacuation_in_progress()) {
151 // Concurrently evacuate
152 entry_evacuate();
153 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false;
154
155 // Perform update-refs phase.
156 vmop_entry_init_updaterefs();
157 entry_updaterefs();
158 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
159
160 // Concurrent update thread roots
161 entry_update_thread_roots();
162 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
163
164 vmop_entry_final_updaterefs();
165
166 // Update references freed up collection set, kick the cleanup to reclaim the space.
167 entry_cleanup_complete();
168 } else {
169 vmop_entry_final_roots();
170 }
171
172 return true;
173 }
174
175 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
176 ShenandoahHeap* const heap = ShenandoahHeap::heap();
177 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
178 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
179
180 heap->try_inject_alloc_failure();
181 VM_ShenandoahInitMark op(this);
182 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
183 }
184
185 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
186 ShenandoahHeap* const heap = ShenandoahHeap::heap();
187 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
188 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
189
190 heap->try_inject_alloc_failure();
191 VM_ShenandoahFinalMarkStartEvac op(this);
192 VMThread::execute(&op); // jump to entry_final_mark under safepoint
193 }
194
195 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
196 ShenandoahHeap* const heap = ShenandoahHeap::heap();
197 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
198 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
199
200 heap->try_inject_alloc_failure();
201 VM_ShenandoahInitUpdateRefs op(this);
202 VMThread::execute(&op);
203 }
204
205 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
206 ShenandoahHeap* const heap = ShenandoahHeap::heap();
207 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
208 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
209
210 heap->try_inject_alloc_failure();
211 VM_ShenandoahFinalUpdateRefs op(this);
212 VMThread::execute(&op);
213 }
214
215 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
216 ShenandoahHeap* const heap = ShenandoahHeap::heap();
217 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
218 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
219
220 // This phase does not use workers, no need for setup
221 heap->try_inject_alloc_failure();
222 VM_ShenandoahFinalRoots op(this);
223 VMThread::execute(&op);
224 }
225
226 void ShenandoahConcurrentGC::entry_init_mark() {
227 const char* msg = init_mark_event_message();
228 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
229 EventMark em("%s", msg);
230
231 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
232 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
233 "init marking");
234
235 op_init_mark();
236 }
237
238 void ShenandoahConcurrentGC::entry_final_mark() {
239 const char* msg = final_mark_event_message();
240 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
241 EventMark em("%s", msg);
242
243 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
244 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
245 "final marking");
246
247 op_final_mark();
248 }
249
250 void ShenandoahConcurrentGC::entry_init_updaterefs() {
251 static const char* msg = "Pause Init Update Refs";
252 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
253 EventMark em("%s", msg);
254
255 // No workers used in this phase, no setup required
256 op_init_updaterefs();
257 }
258
259 void ShenandoahConcurrentGC::entry_final_updaterefs() {
260 static const char* msg = "Pause Final Update Refs";
261 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
262 EventMark em("%s", msg);
263
264 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
265 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
266 "final reference update");
267
268 op_final_updaterefs();
269 }
270
271 void ShenandoahConcurrentGC::entry_final_roots() {
272 static const char* msg = "Pause Final Roots";
273 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
274 EventMark em("%s", msg);
275
276 op_final_roots();
277 }
278
279 void ShenandoahConcurrentGC::entry_reset() {
280 ShenandoahHeap* const heap = ShenandoahHeap::heap();
281 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
282 static const char* msg = "Concurrent reset";
283 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
284 EventMark em("%s", msg);
285
286 ShenandoahWorkerScope scope(heap->workers(),
287 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
288 "concurrent reset");
289
290 heap->try_inject_alloc_failure();
291 op_reset();
292 }
293
294 void ShenandoahConcurrentGC::entry_mark_roots() {
295 ShenandoahHeap* const heap = ShenandoahHeap::heap();
296 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
297 const char* msg = "Concurrent marking roots";
298 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
299 EventMark em("%s", msg);
300
301 ShenandoahWorkerScope scope(heap->workers(),
302 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
303 "concurrent marking roots");
304
305 heap->try_inject_alloc_failure();
306 op_mark_roots();
307 }
308
309 void ShenandoahConcurrentGC::entry_mark() {
310 ShenandoahHeap* const heap = ShenandoahHeap::heap();
311 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
312 const char* msg = conc_mark_event_message();
313 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
314 EventMark em("%s", msg);
315
316 ShenandoahWorkerScope scope(heap->workers(),
317 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
318 "concurrent marking");
319
320 heap->try_inject_alloc_failure();
321 op_mark();
322 }
323
324 void ShenandoahConcurrentGC::entry_thread_roots() {
325 ShenandoahHeap* const heap = ShenandoahHeap::heap();
326 static const char* msg = "Concurrent thread roots";
327 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
328 EventMark em("%s", msg);
329
330 ShenandoahWorkerScope scope(heap->workers(),
331 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
332 msg);
333
334 heap->try_inject_alloc_failure();
335 op_thread_roots();
336 }
337
338 void ShenandoahConcurrentGC::entry_weak_refs() {
339 ShenandoahHeap* const heap = ShenandoahHeap::heap();
340 static const char* msg = "Concurrent weak references";
341 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
342 EventMark em("%s", msg);
343
344 ShenandoahWorkerScope scope(heap->workers(),
345 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
346 "concurrent weak references");
347
348 heap->try_inject_alloc_failure();
349 op_weak_refs();
350 }
351
352 void ShenandoahConcurrentGC::entry_weak_roots() {
353 ShenandoahHeap* const heap = ShenandoahHeap::heap();
354 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
355 static const char* msg = "Concurrent weak roots";
356 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
357 EventMark em("%s", msg);
358
359 ShenandoahWorkerScope scope(heap->workers(),
360 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
361 "concurrent weak root");
362
363 heap->try_inject_alloc_failure();
364 op_weak_roots();
365 }
366
367 void ShenandoahConcurrentGC::entry_class_unloading() {
368 ShenandoahHeap* const heap = ShenandoahHeap::heap();
369 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
370 static const char* msg = "Concurrent class unloading";
371 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
372 EventMark em("%s", msg);
373
374 ShenandoahWorkerScope scope(heap->workers(),
375 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
376 "concurrent class unloading");
377
378 heap->try_inject_alloc_failure();
379 op_class_unloading();
380 }
381
382 void ShenandoahConcurrentGC::entry_strong_roots() {
383 ShenandoahHeap* const heap = ShenandoahHeap::heap();
384 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
385 static const char* msg = "Concurrent strong roots";
386 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
387 EventMark em("%s", msg);
388
389 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
390
391 ShenandoahWorkerScope scope(heap->workers(),
392 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
393 "concurrent strong root");
394
395 heap->try_inject_alloc_failure();
396 op_strong_roots();
397 }
398
399 void ShenandoahConcurrentGC::entry_cleanup_early() {
400 ShenandoahHeap* const heap = ShenandoahHeap::heap();
401 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
402 static const char* msg = "Concurrent cleanup";
403 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
404 EventMark em("%s", msg);
405
406 // This phase does not use workers, no need for setup
407 heap->try_inject_alloc_failure();
408 op_cleanup_early();
409 }
410
411 void ShenandoahConcurrentGC::entry_evacuate() {
412 ShenandoahHeap* const heap = ShenandoahHeap::heap();
413 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
414
415 static const char* msg = "Concurrent evacuation";
416 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
417 EventMark em("%s", msg);
418
419 ShenandoahWorkerScope scope(heap->workers(),
420 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
421 "concurrent evacuation");
422
423 heap->try_inject_alloc_failure();
424 op_evacuate();
425 }
426
427 void ShenandoahConcurrentGC::entry_update_thread_roots() {
428 ShenandoahHeap* const heap = ShenandoahHeap::heap();
429 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
430
431 static const char* msg = "Concurrent update thread roots";
432 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
433 EventMark em("%s", msg);
434
435 // No workers used in this phase, no setup required
436 heap->try_inject_alloc_failure();
437 op_update_thread_roots();
438 }
439
440 void ShenandoahConcurrentGC::entry_updaterefs() {
441 ShenandoahHeap* const heap = ShenandoahHeap::heap();
442 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
443 static const char* msg = "Concurrent update references";
444 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
445 EventMark em("%s", msg);
446
447 ShenandoahWorkerScope scope(heap->workers(),
448 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
449 "concurrent reference update");
450
451 heap->try_inject_alloc_failure();
452 op_updaterefs();
453 }
454
455 void ShenandoahConcurrentGC::entry_cleanup_complete() {
456 ShenandoahHeap* const heap = ShenandoahHeap::heap();
457 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
458 static const char* msg = "Concurrent cleanup";
459 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
460 EventMark em("%s", msg);
461
462 // This phase does not use workers, no need for setup
463 heap->try_inject_alloc_failure();
464 op_cleanup_complete();
465 }
466
467 void ShenandoahConcurrentGC::op_reset() {
468 ShenandoahHeap* const heap = ShenandoahHeap::heap();
469 if (ShenandoahPacing) {
470 heap->pacer()->setup_for_reset();
471 }
472
473 heap->prepare_gc();
474 }
475
476 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
477 private:
478 ShenandoahMarkingContext* const _ctx;
479 public:
480 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
481
482 void heap_region_do(ShenandoahHeapRegion* r) {
483 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
484 if (r->is_active()) {
485 // Check if region needs updating its TAMS. We have updated it already during concurrent
486 // reset, so it is very likely we don't need to do another write here.
487 if (_ctx->top_at_mark_start(r) != r->top()) {
488 _ctx->capture_top_at_mark_start(r);
489 }
490 } else {
491 assert(_ctx->top_at_mark_start(r) == r->top(),
492 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
493 }
494 }
495
496 bool is_thread_safe() { return true; }
497 };
498
499 void ShenandoahConcurrentGC::op_init_mark() {
500 ShenandoahHeap* const heap = ShenandoahHeap::heap();
501 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
502 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
503
504 assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap");
505 assert(!heap->marking_context()->is_complete(), "should not be complete");
506 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
507
508 if (ShenandoahVerify) {
509 heap->verifier()->verify_before_concmark();
510 }
511
512 if (VerifyBeforeGC) {
513 Universe::verify();
514 }
515
516 heap->set_concurrent_mark_in_progress(true);
517
518 {
519 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
520 ShenandoahInitMarkUpdateRegionStateClosure cl;
521 heap->parallel_heap_region_iterate(&cl);
522 }
523
524 // Weak reference processing
525 ShenandoahReferenceProcessor* rp = heap->ref_processor();
526 rp->reset_thread_locals();
527 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
528
529 // Make above changes visible to worker threads
530 OrderAccess::fence();
531 // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
532 // we need to make sure that all its metadata are marked. alternative is to remark
533 // thread roots at final mark pause, but it can be potential latency killer.
534 if (heap->unload_classes()) {
535 ShenandoahCodeRoots::arm_nmethods();
536 }
537
538 ShenandoahStackWatermark::change_epoch_id();
539 if (ShenandoahPacing) {
540 heap->pacer()->setup_for_mark();
541 }
542 }
543
544 void ShenandoahConcurrentGC::op_mark_roots() {
545 _mark.mark_concurrent_roots();
546 }
547
548 void ShenandoahConcurrentGC::op_mark() {
549 _mark.concurrent_mark();
550 }
551
552 void ShenandoahConcurrentGC::op_final_mark() {
553 ShenandoahHeap* const heap = ShenandoahHeap::heap();
554 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
555 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
556
557 if (ShenandoahVerify) {
558 heap->verifier()->verify_roots_no_forwarded();
559 }
560
561 if (!heap->cancelled_gc()) {
562 _mark.finish_mark();
563 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
564
565 // Notify JVMTI that the tagmap table will need cleaning.
566 JvmtiTagMap::set_needs_cleaning();
567
568 heap->prepare_regions_and_collection_set(true /*concurrent*/);
569
570 // Has to be done after cset selection
571 heap->prepare_concurrent_roots();
572
573 if (!heap->collection_set()->is_empty()) {
574 if (ShenandoahVerify) {
575 heap->verifier()->verify_before_evacuation();
576 }
577
578 heap->set_evacuation_in_progress(true);
579 // From here on, we need to update references.
580 heap->set_has_forwarded_objects(true);
581
582 // Verify before arming for concurrent processing.
583 // Otherwise, verification can trigger stack processing.
584 if (ShenandoahVerify) {
585 heap->verifier()->verify_during_evacuation();
586 }
587
588 // Arm nmethods/stack for concurrent processing
589 ShenandoahCodeRoots::arm_nmethods();
590 ShenandoahStackWatermark::change_epoch_id();
591
592 // Notify JVMTI that oops are changed.
593 JvmtiTagMap::set_needs_rehashing();
594
595 if (ShenandoahPacing) {
596 heap->pacer()->setup_for_evac();
597 }
598 } else {
599 if (ShenandoahVerify) {
600 heap->verifier()->verify_after_concmark();
601 }
602
603 if (VerifyAfterGC) {
604 Universe::verify();
605 }
606 }
607 }
608 }
609
610 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
611 private:
612 OopClosure* const _oops;
613
614 public:
615 ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
616 void do_thread(Thread* thread);
617 };
618
619 ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
620 _oops(oops) {
621 }
622
623 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
624 JavaThread* const jt = JavaThread::cast(thread);
625 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
626 }
627
628 class ShenandoahConcurrentEvacUpdateThreadTask : public AbstractGangTask {
629 private:
630 ShenandoahJavaThreadsIterator _java_threads;
631
632 public:
633 ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
634 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
635 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
636 }
637
638 void work(uint worker_id) {
639 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
640 // Otherwise, may deadlock with watermark lock
641 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
642 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
643 _java_threads.threads_do(&thr_cl, worker_id);
644 }
645 };
646
647 void ShenandoahConcurrentGC::op_thread_roots() {
648 ShenandoahHeap* const heap = ShenandoahHeap::heap();
649 assert(heap->is_evacuation_in_progress(), "Checked by caller");
650 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
651 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
652 heap->workers()->run_task(&task);
653 }
654
655 void ShenandoahConcurrentGC::op_weak_refs() {
656 ShenandoahHeap* const heap = ShenandoahHeap::heap();
657 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
658 // Concurrent weak refs processing
659 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
660 ShenandoahBreakpoint::at_after_reference_processing_started();
661 heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
662 }
663
664 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
665 private:
666 ShenandoahHeap* const _heap;
667 ShenandoahMarkingContext* const _mark_context;
668 bool _evac_in_progress;
669 Thread* const _thread;
670
671 public:
672 ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
673 void do_oop(oop* p);
674 void do_oop(narrowOop* p);
675 };
676
677 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
678 _heap(ShenandoahHeap::heap()),
679 _mark_context(ShenandoahHeap::heap()->marking_context()),
680 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
681 _thread(Thread::current()) {
682 }
683
684 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
685 const oop obj = RawAccess<>::oop_load(p);
686 if (!CompressedOops::is_null(obj)) {
687 if (!_mark_context->is_marked(obj)) {
688 shenandoah_assert_correct(p, obj);
689 ShenandoahHeap::atomic_clear_oop(p, obj);
690 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
691 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
692 if (resolved == obj) {
693 resolved = _heap->evacuate_object(obj, _thread);
694 }
695 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
696 assert(_heap->cancelled_gc() ||
697 _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
698 "Sanity");
699 }
700 }
701 }
702
703 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
704 ShouldNotReachHere();
705 }
706
707 class ShenandoahIsCLDAliveClosure : public CLDClosure {
708 public:
709 void do_cld(ClassLoaderData* cld) {
710 cld->is_alive();
711 }
712 };
713
714 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
715 public:
716 void do_nmethod(nmethod* n) {
717 n->is_unloading();
718 }
719 };
720
721 // This task not only evacuates/updates marked weak roots, but also "NULL"
722 // dead weak roots.
723 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
724 private:
725 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
726
727 // Roots related to concurrent class unloading
728 ShenandoahClassLoaderDataRoots<true /* concurrent */, true /* single thread*/>
729 _cld_roots;
730 ShenandoahConcurrentNMethodIterator _nmethod_itr;
731 ShenandoahPhaseTimings::Phase _phase;
732
733 public:
734 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
735 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
736 _vm_roots(phase),
737 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()),
738 _nmethod_itr(ShenandoahCodeRoots::table()),
739 _phase(phase) {
740 if (ShenandoahHeap::heap()->unload_classes()) {
741 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
742 _nmethod_itr.nmethods_do_begin();
743 }
744 }
745
746 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
747 if (ShenandoahHeap::heap()->unload_classes()) {
748 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
749 _nmethod_itr.nmethods_do_end();
750 }
751 // Notify runtime data structures of potentially dead oops
752 _vm_roots.report_num_dead();
753 }
754
755 void work(uint worker_id) {
756 ShenandoahConcurrentWorkerSession worker_session(worker_id);
757 {
758 ShenandoahEvacOOMScope oom;
759 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
760 // may race against OopStorage::release() calls.
761 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
762 _vm_roots.oops_do(&cl, worker_id);
763 }
764
765 // If we are going to perform concurrent class unloading later on, we need to
766 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
767 // can cleanup immediate garbage sooner.
768 if (ShenandoahHeap::heap()->unload_classes()) {
769 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
770 // CLD's holder or evacuate it.
771 {
772 ShenandoahIsCLDAliveClosure is_cld_alive;
773 _cld_roots.cld_do(&is_cld_alive, worker_id);
774 }
775
776 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
777 // The closure calls nmethod->is_unloading(). The is_unloading
778 // state is cached, therefore, during concurrent class unloading phase,
779 // we will not touch the metadata of unloading nmethods
780 {
781 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
782 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
783 _nmethod_itr.nmethods_do(&is_nmethod_alive);
784 }
785 }
786 }
787 };
788
789 void ShenandoahConcurrentGC::op_weak_roots() {
790 ShenandoahHeap* const heap = ShenandoahHeap::heap();
791 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
792 // Concurrent weak root processing
793 {
794 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
795 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
796 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
797 heap->workers()->run_task(&task);
798 }
799
800 // Perform handshake to flush out dead oops
801 {
802 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
803 heap->rendezvous_threads();
804 }
805 }
806
807 void ShenandoahConcurrentGC::op_class_unloading() {
808 ShenandoahHeap* const heap = ShenandoahHeap::heap();
809 assert (heap->is_concurrent_weak_root_in_progress() &&
810 heap->unload_classes(),
811 "Checked by caller");
812 heap->do_class_unloading();
813 }
814
815 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
816 private:
817 BarrierSetNMethod* const _bs;
818 ShenandoahEvacuateUpdateMetadataClosure<> _cl;
819
820 public:
821 ShenandoahEvacUpdateCodeCacheClosure() :
822 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
823 _cl() {
824 }
825
826 void do_nmethod(nmethod* n) {
827 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
828 ShenandoahReentrantLocker locker(data->lock());
829 // Setup EvacOOM scope below reentrant lock to avoid deadlock with
830 // nmethod_entry_barrier
831 ShenandoahEvacOOMScope oom;
832 data->oops_do(&_cl, true/*fix relocation*/);
833 _bs->disarm(n);
834 }
835 };
836
837 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
838 private:
839 ShenandoahPhaseTimings::Phase _phase;
840 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
841 ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
842 ShenandoahConcurrentNMethodIterator _nmethod_itr;
843
844 public:
845 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
846 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
847 _phase(phase),
848 _vm_roots(phase),
849 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()),
850 _nmethod_itr(ShenandoahCodeRoots::table()) {
851 if (!ShenandoahHeap::heap()->unload_classes()) {
852 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
853 _nmethod_itr.nmethods_do_begin();
854 }
855 }
856
857 ~ShenandoahConcurrentRootsEvacUpdateTask() {
858 if (!ShenandoahHeap::heap()->unload_classes()) {
859 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
860 _nmethod_itr.nmethods_do_end();
861 }
862 }
863
864 void work(uint worker_id) {
865 ShenandoahConcurrentWorkerSession worker_session(worker_id);
866 {
867 ShenandoahEvacOOMScope oom;
868 {
869 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
870 // may race against OopStorage::release() calls.
871 ShenandoahContextEvacuateUpdateRootsClosure cl;
872 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
873 }
874
875 {
876 ShenandoahEvacuateUpdateMetadataClosure<> cl;
877 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
878 _cld_roots.cld_do(&clds, worker_id);
879 }
880 }
881
882 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
883 if (!ShenandoahHeap::heap()->unload_classes()) {
884 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
885 ShenandoahEvacUpdateCodeCacheClosure cl;
886 _nmethod_itr.nmethods_do(&cl);
887 }
888 }
889 };
890
891 void ShenandoahConcurrentGC::op_strong_roots() {
892 ShenandoahHeap* const heap = ShenandoahHeap::heap();
893 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
894 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
895 heap->workers()->run_task(&task);
896 heap->set_concurrent_strong_root_in_progress(false);
897 }
898
899 void ShenandoahConcurrentGC::op_cleanup_early() {
900 ShenandoahHeap::heap()->free_set()->recycle_trash();
901 }
902
903 void ShenandoahConcurrentGC::op_evacuate() {
904 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
905 }
906
907 void ShenandoahConcurrentGC::op_init_updaterefs() {
908 ShenandoahHeap* const heap = ShenandoahHeap::heap();
909 heap->set_evacuation_in_progress(false);
910 heap->set_concurrent_weak_root_in_progress(false);
911 heap->prepare_update_heap_references(true /*concurrent*/);
912 heap->set_update_refs_in_progress(true);
913
914 if (ShenandoahPacing) {
915 heap->pacer()->setup_for_updaterefs();
916 }
917 }
918
919 void ShenandoahConcurrentGC::op_updaterefs() {
920 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
921 }
922
923 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
924 private:
925 ShenandoahUpdateRefsClosure _cl;
926 public:
927 ShenandoahUpdateThreadClosure();
928 void do_thread(Thread* thread);
929 };
930
931 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
932 HandshakeClosure("Shenandoah Update Thread Roots") {
933 }
934
935 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
936 if (thread->is_Java_thread()) {
937 JavaThread* jt = JavaThread::cast(thread);
938 ResourceMark rm;
939 jt->oops_do(&_cl, NULL);
940 }
941 }
942
943 void ShenandoahConcurrentGC::op_update_thread_roots() {
944 ShenandoahUpdateThreadClosure cl;
945 Handshake::execute(&cl);
946 }
947
948 void ShenandoahConcurrentGC::op_final_updaterefs() {
949 ShenandoahHeap* const heap = ShenandoahHeap::heap();
950 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
951 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
952
953 heap->finish_concurrent_roots();
954
955 // Clear cancelled GC, if set. On cancellation path, the block before would handle
956 // everything.
957 if (heap->cancelled_gc()) {
958 heap->clear_cancelled_gc();
959 }
960
961 // Has to be done before cset is clear
962 if (ShenandoahVerify) {
963 heap->verifier()->verify_roots_in_to_space();
964 }
965
966 heap->update_heap_region_states(true /*concurrent*/);
967
968 heap->set_update_refs_in_progress(false);
969 heap->set_has_forwarded_objects(false);
970
971 if (ShenandoahVerify) {
972 heap->verifier()->verify_after_updaterefs();
973 }
974
975 if (VerifyAfterGC) {
976 Universe::verify();
977 }
978
979 heap->rebuild_free_set(true /*concurrent*/);
980 }
981
982 void ShenandoahConcurrentGC::op_final_roots() {
983 ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
984 }
985
986 void ShenandoahConcurrentGC::op_cleanup_complete() {
987 ShenandoahHeap::heap()->free_set()->recycle_trash();
988 }
989
990 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
991 if (ShenandoahHeap::heap()->cancelled_gc()) {
992 _degen_point = point;
993 return true;
994 }
995 return false;
996 }
997
998 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
999 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1000 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1001 if (heap->unload_classes()) {
1002 return "Pause Init Mark (unload classes)";
1003 } else {
1004 return "Pause Init Mark";
1005 }
1006 }
1007
1008 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1009 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1010 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1011 if (heap->unload_classes()) {
1012 return "Pause Final Mark (unload classes)";
1013 } else {
1014 return "Pause Final Mark";
1015 }
1016 }
1017
1018 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1019 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1020 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1021 if (heap->unload_classes()) {
1022 return "Concurrent marking (unload classes)";
1023 } else {
1024 return "Concurrent marking";
1025 }
1026 }