1 /*
2 * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/barrierSetNMethod.hpp"
28 #include "gc/shared/collectorCounters.hpp"
29 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
30 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
31 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahLock.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
36 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
37 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
38 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
39 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
40 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "gc/shenandoah/shenandoahVerifier.hpp"
43 #include "gc/shenandoah/shenandoahVMOperations.hpp"
44 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
46 #include "memory/allocation.hpp"
47 #include "prims/jvmtiTagMap.hpp"
48 #include "runtime/vmThread.hpp"
49 #include "utilities/events.hpp"
50
51 // Breakpoint support
52 class ShenandoahBreakpointGCScope : public StackObj {
53 private:
54 const GCCause::Cause _cause;
55 public:
56 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
57 if (cause == GCCause::_wb_breakpoint) {
58 ShenandoahBreakpoint::start_gc();
59 ShenandoahBreakpoint::at_before_gc();
60 }
61 }
62
63 ~ShenandoahBreakpointGCScope() {
64 if (_cause == GCCause::_wb_breakpoint) {
65 ShenandoahBreakpoint::at_after_gc();
66 }
67 }
68 };
69
70 class ShenandoahBreakpointMarkScope : public StackObj {
71 private:
72 const GCCause::Cause _cause;
73 public:
74 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
75 if (_cause == GCCause::_wb_breakpoint) {
76 ShenandoahBreakpoint::at_after_marking_started();
77 }
78 }
79
80 ~ShenandoahBreakpointMarkScope() {
81 if (_cause == GCCause::_wb_breakpoint) {
82 ShenandoahBreakpoint::at_before_marking_completed();
83 }
84 }
85 };
86
87 ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
88 _mark(),
89 _degen_point(ShenandoahDegenPoint::_degenerated_unset) {
90 }
91
92 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
93 return _degen_point;
94 }
95
96 void ShenandoahConcurrentGC::cancel() {
97 ShenandoahConcurrentMark::cancel();
98 }
99
100 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
101 ShenandoahHeap* const heap = ShenandoahHeap::heap();
102 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
103
104 // Reset for upcoming marking
105 entry_reset();
106
107 // Start initial mark under STW
108 vmop_entry_init_mark();
109
110 {
111 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
112 // Concurrent mark roots
113 entry_mark_roots();
114 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false;
115
116 // Continue concurrent mark
117 entry_mark();
118 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false;
119 }
120
121 // Complete marking under STW, and start evacuation
122 vmop_entry_final_mark();
123
124 // Concurrent stack processing
125 if (heap->is_evacuation_in_progress()) {
126 entry_thread_roots();
127 }
128
129 // Process weak roots that might still point to regions that would be broken by cleanup
130 if (heap->is_concurrent_weak_root_in_progress()) {
131 entry_weak_refs();
132 entry_weak_roots();
133 }
134
135 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
136 // the space. This would be the last action if there is nothing to evacuate.
137 entry_cleanup_early();
138
139 {
140 ShenandoahHeapLocker locker(heap->lock());
141 heap->free_set()->log_status();
142 }
143
144 // Perform concurrent class unloading
145 if (heap->unload_classes() &&
146 heap->is_concurrent_weak_root_in_progress()) {
147 entry_class_unloading();
148 }
149
150 // Processing strong roots
151 // This may be skipped if there is nothing to update/evacuate.
152 // If so, strong_root_in_progress would be unset.
153 if (heap->is_concurrent_strong_root_in_progress()) {
154 entry_strong_roots();
155 }
156
157 // Continue the cycle with evacuation and optional update-refs.
158 // This may be skipped if there is nothing to evacuate.
159 // If so, evac_in_progress would be unset by collection set preparation code.
160 if (heap->is_evacuation_in_progress()) {
161 // Concurrently evacuate
162 entry_evacuate();
163 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false;
164
165 // Perform update-refs phase.
166 vmop_entry_init_updaterefs();
167 entry_updaterefs();
168 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
169
170 // Concurrent update thread roots
171 entry_update_thread_roots();
172 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
173
174 vmop_entry_final_updaterefs();
175
176 // Update references freed up collection set, kick the cleanup to reclaim the space.
177 entry_cleanup_complete();
178 } else {
179 vmop_entry_final_roots();
180 }
181
182 return true;
183 }
184
185 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
186 ShenandoahHeap* const heap = ShenandoahHeap::heap();
187 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
188 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
189
190 heap->try_inject_alloc_failure();
191 VM_ShenandoahInitMark op(this);
192 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
193 }
194
195 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
196 ShenandoahHeap* const heap = ShenandoahHeap::heap();
197 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
198 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
199
200 heap->try_inject_alloc_failure();
201 VM_ShenandoahFinalMarkStartEvac op(this);
202 VMThread::execute(&op); // jump to entry_final_mark under safepoint
203 }
204
205 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
206 ShenandoahHeap* const heap = ShenandoahHeap::heap();
207 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
208 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
209
210 heap->try_inject_alloc_failure();
211 VM_ShenandoahInitUpdateRefs op(this);
212 VMThread::execute(&op);
213 }
214
215 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
216 ShenandoahHeap* const heap = ShenandoahHeap::heap();
217 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
218 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
219
220 heap->try_inject_alloc_failure();
221 VM_ShenandoahFinalUpdateRefs op(this);
222 VMThread::execute(&op);
223 }
224
225 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
226 ShenandoahHeap* const heap = ShenandoahHeap::heap();
227 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
228 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
229
230 // This phase does not use workers, no need for setup
231 heap->try_inject_alloc_failure();
232 VM_ShenandoahFinalRoots op(this);
233 VMThread::execute(&op);
234 }
235
236 void ShenandoahConcurrentGC::entry_init_mark() {
237 const char* msg = init_mark_event_message();
238 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
239 EventMark em("%s", msg);
240
241 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
242 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
243 "init marking");
244
245 op_init_mark();
246 }
247
248 void ShenandoahConcurrentGC::entry_final_mark() {
249 const char* msg = final_mark_event_message();
250 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
251 EventMark em("%s", msg);
252
253 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
254 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
255 "final marking");
256
257 op_final_mark();
258 }
259
260 void ShenandoahConcurrentGC::entry_init_updaterefs() {
261 static const char* msg = "Pause Init Update Refs";
262 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
263 EventMark em("%s", msg);
264
265 // No workers used in this phase, no setup required
266 op_init_updaterefs();
267 }
268
269 void ShenandoahConcurrentGC::entry_final_updaterefs() {
270 static const char* msg = "Pause Final Update Refs";
271 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
272 EventMark em("%s", msg);
273
274 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
275 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
276 "final reference update");
277
278 op_final_updaterefs();
279 }
280
281 void ShenandoahConcurrentGC::entry_final_roots() {
282 static const char* msg = "Pause Final Roots";
283 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
284 EventMark em("%s", msg);
285
286 op_final_roots();
287 }
288
289 void ShenandoahConcurrentGC::entry_reset() {
290 ShenandoahHeap* const heap = ShenandoahHeap::heap();
291 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
292 static const char* msg = "Concurrent reset";
293 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
294 EventMark em("%s", msg);
295
296 ShenandoahWorkerScope scope(heap->workers(),
297 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
298 "concurrent reset");
299
300 heap->try_inject_alloc_failure();
301 op_reset();
302 }
303
304 void ShenandoahConcurrentGC::entry_mark_roots() {
305 ShenandoahHeap* const heap = ShenandoahHeap::heap();
306 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
307 const char* msg = "Concurrent marking roots";
308 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
309 EventMark em("%s", msg);
310
311 ShenandoahWorkerScope scope(heap->workers(),
312 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
313 "concurrent marking roots");
314
315 heap->try_inject_alloc_failure();
316 op_mark_roots();
317 }
318
319 void ShenandoahConcurrentGC::entry_mark() {
320 ShenandoahHeap* const heap = ShenandoahHeap::heap();
321 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
322 const char* msg = conc_mark_event_message();
323 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
324 EventMark em("%s", msg);
325
326 ShenandoahWorkerScope scope(heap->workers(),
327 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
328 "concurrent marking");
329
330 heap->try_inject_alloc_failure();
331 op_mark();
332 }
333
334 void ShenandoahConcurrentGC::entry_thread_roots() {
335 ShenandoahHeap* const heap = ShenandoahHeap::heap();
336 static const char* msg = "Concurrent thread roots";
337 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
338 EventMark em("%s", msg);
339
340 ShenandoahWorkerScope scope(heap->workers(),
341 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
342 msg);
343
344 heap->try_inject_alloc_failure();
345 op_thread_roots();
346 }
347
348 void ShenandoahConcurrentGC::entry_weak_refs() {
349 ShenandoahHeap* const heap = ShenandoahHeap::heap();
350 static const char* msg = "Concurrent weak references";
351 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
352 EventMark em("%s", msg);
353
354 ShenandoahWorkerScope scope(heap->workers(),
355 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
356 "concurrent weak references");
357
358 heap->try_inject_alloc_failure();
359 op_weak_refs();
360 }
361
362 void ShenandoahConcurrentGC::entry_weak_roots() {
363 ShenandoahHeap* const heap = ShenandoahHeap::heap();
364 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
365 static const char* msg = "Concurrent weak roots";
366 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
367 EventMark em("%s", msg);
368
369 ShenandoahWorkerScope scope(heap->workers(),
370 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
371 "concurrent weak root");
372
373 heap->try_inject_alloc_failure();
374 op_weak_roots();
375 }
376
377 void ShenandoahConcurrentGC::entry_class_unloading() {
378 ShenandoahHeap* const heap = ShenandoahHeap::heap();
379 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
380 static const char* msg = "Concurrent class unloading";
381 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
382 EventMark em("%s", msg);
383
384 ShenandoahWorkerScope scope(heap->workers(),
385 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
386 "concurrent class unloading");
387
388 heap->try_inject_alloc_failure();
389 op_class_unloading();
390 }
391
392 void ShenandoahConcurrentGC::entry_strong_roots() {
393 ShenandoahHeap* const heap = ShenandoahHeap::heap();
394 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
395 static const char* msg = "Concurrent strong roots";
396 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
397 EventMark em("%s", msg);
398
399 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
400
401 ShenandoahWorkerScope scope(heap->workers(),
402 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
403 "concurrent strong root");
404
405 heap->try_inject_alloc_failure();
406 op_strong_roots();
407 }
408
409 void ShenandoahConcurrentGC::entry_cleanup_early() {
410 ShenandoahHeap* const heap = ShenandoahHeap::heap();
411 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
412 static const char* msg = "Concurrent cleanup";
413 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
414 EventMark em("%s", msg);
415
416 // This phase does not use workers, no need for setup
417 heap->try_inject_alloc_failure();
418 op_cleanup_early();
419 }
420
421 void ShenandoahConcurrentGC::entry_evacuate() {
422 ShenandoahHeap* const heap = ShenandoahHeap::heap();
423 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
424
425 static const char* msg = "Concurrent evacuation";
426 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
427 EventMark em("%s", msg);
428
429 ShenandoahWorkerScope scope(heap->workers(),
430 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
431 "concurrent evacuation");
432
433 heap->try_inject_alloc_failure();
434 op_evacuate();
435 }
436
437 void ShenandoahConcurrentGC::entry_update_thread_roots() {
438 ShenandoahHeap* const heap = ShenandoahHeap::heap();
439 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
440
441 static const char* msg = "Concurrent update thread roots";
442 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
443 EventMark em("%s", msg);
444
445 // No workers used in this phase, no setup required
446 heap->try_inject_alloc_failure();
447 op_update_thread_roots();
448 }
449
450 void ShenandoahConcurrentGC::entry_updaterefs() {
451 ShenandoahHeap* const heap = ShenandoahHeap::heap();
452 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
453 static const char* msg = "Concurrent update references";
454 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
455 EventMark em("%s", msg);
456
457 ShenandoahWorkerScope scope(heap->workers(),
458 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
459 "concurrent reference update");
460
461 heap->try_inject_alloc_failure();
462 op_updaterefs();
463 }
464
465 void ShenandoahConcurrentGC::entry_cleanup_complete() {
466 ShenandoahHeap* const heap = ShenandoahHeap::heap();
467 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
468 static const char* msg = "Concurrent cleanup";
469 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
470 EventMark em("%s", msg);
471
472 // This phase does not use workers, no need for setup
473 heap->try_inject_alloc_failure();
474 op_cleanup_complete();
475 }
476
477 void ShenandoahConcurrentGC::op_reset() {
478 ShenandoahHeap* const heap = ShenandoahHeap::heap();
479 if (ShenandoahPacing) {
480 heap->pacer()->setup_for_reset();
481 }
482
483 heap->prepare_gc();
484 }
485
486 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
487 private:
488 ShenandoahMarkingContext* const _ctx;
489 public:
490 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
491
492 void heap_region_do(ShenandoahHeapRegion* r) {
493 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
494 if (r->is_active()) {
495 // Check if region needs updating its TAMS. We have updated it already during concurrent
496 // reset, so it is very likely we don't need to do another write here.
497 if (_ctx->top_at_mark_start(r) != r->top()) {
498 _ctx->capture_top_at_mark_start(r);
499 }
500 } else {
501 assert(_ctx->top_at_mark_start(r) == r->top(),
502 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
503 }
504 }
505
506 bool is_thread_safe() { return true; }
507 };
508
509 void ShenandoahConcurrentGC::op_init_mark() {
510 ShenandoahHeap* const heap = ShenandoahHeap::heap();
511 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
512 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
513
514 assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap");
515 assert(!heap->marking_context()->is_complete(), "should not be complete");
516 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
517
518 if (ShenandoahVerify) {
519 heap->verifier()->verify_before_concmark();
520 }
521
522 if (VerifyBeforeGC) {
523 Universe::verify();
524 }
525
526 heap->set_concurrent_mark_in_progress(true);
527
528 {
529 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
530 ShenandoahInitMarkUpdateRegionStateClosure cl;
531 heap->parallel_heap_region_iterate(&cl);
532 }
533
534 // Weak reference processing
535 ShenandoahReferenceProcessor* rp = heap->ref_processor();
536 rp->reset_thread_locals();
537 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
538
539 // Make above changes visible to worker threads
540 OrderAccess::fence();
541 // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
542 // we need to make sure that all its metadata are marked. alternative is to remark
543 // thread roots at final mark pause, but it can be potential latency killer.
544 if (heap->unload_classes()) {
545 ShenandoahCodeRoots::arm_nmethods();
546 }
547
548 ShenandoahStackWatermark::change_epoch_id();
549 if (ShenandoahPacing) {
550 heap->pacer()->setup_for_mark();
551 }
552 }
553
554 void ShenandoahConcurrentGC::op_mark_roots() {
555 _mark.mark_concurrent_roots();
556 }
557
558 void ShenandoahConcurrentGC::op_mark() {
559 _mark.concurrent_mark();
560 }
561
562 void ShenandoahConcurrentGC::op_final_mark() {
563 ShenandoahHeap* const heap = ShenandoahHeap::heap();
564 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
565 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
566
567 if (ShenandoahVerify) {
568 heap->verifier()->verify_roots_no_forwarded();
569 }
570
571 if (!heap->cancelled_gc()) {
572 _mark.finish_mark();
573 assert(!heap->cancelled_gc(), "STW mark cannot OOM");
574
575 // Notify JVMTI that the tagmap table will need cleaning.
576 JvmtiTagMap::set_needs_cleaning();
577
578 heap->prepare_regions_and_collection_set(true /*concurrent*/);
579
580 // Has to be done after cset selection
581 heap->prepare_concurrent_roots();
582
583 if (!heap->collection_set()->is_empty()) {
584 if (ShenandoahVerify) {
585 heap->verifier()->verify_before_evacuation();
586 }
587
588 heap->set_evacuation_in_progress(true);
589 // From here on, we need to update references.
590 heap->set_has_forwarded_objects(true);
591
592 // Verify before arming for concurrent processing.
593 // Otherwise, verification can trigger stack processing.
594 if (ShenandoahVerify) {
595 heap->verifier()->verify_during_evacuation();
596 }
597
598 // Arm nmethods/stack for concurrent processing
599 ShenandoahCodeRoots::arm_nmethods();
600 ShenandoahStackWatermark::change_epoch_id();
601
602 // Notify JVMTI that oops are changed.
603 JvmtiTagMap::set_needs_rehashing();
604
605 if (ShenandoahPacing) {
606 heap->pacer()->setup_for_evac();
607 }
608 } else {
609 if (ShenandoahVerify) {
610 heap->verifier()->verify_after_concmark();
611 }
612
613 if (VerifyAfterGC) {
614 Universe::verify();
615 }
616 }
617 }
618 }
619
620 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
621 private:
622 OopClosure* const _oops;
623
624 public:
625 ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
626 void do_thread(Thread* thread);
627 };
628
629 ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
630 _oops(oops) {
631 }
632
633 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
634 JavaThread* const jt = JavaThread::cast(thread);
635 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
636 }
637
638 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
639 private:
640 ShenandoahJavaThreadsIterator _java_threads;
641
642 public:
643 ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
644 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
645 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
646 }
647
648 void work(uint worker_id) {
649 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
650 // Otherwise, may deadlock with watermark lock
651 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
652 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
653 _java_threads.threads_do(&thr_cl, worker_id);
654 }
655 };
656
657 void ShenandoahConcurrentGC::op_thread_roots() {
658 ShenandoahHeap* const heap = ShenandoahHeap::heap();
659 assert(heap->is_evacuation_in_progress(), "Checked by caller");
660 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
661 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
662 heap->workers()->run_task(&task);
663 }
664
665 void ShenandoahConcurrentGC::op_weak_refs() {
666 ShenandoahHeap* const heap = ShenandoahHeap::heap();
667 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
668 // Concurrent weak refs processing
669 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
670 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
671 ShenandoahBreakpoint::at_after_reference_processing_started();
672 }
673 heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
674 }
675
676 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
677 private:
678 ShenandoahHeap* const _heap;
679 ShenandoahMarkingContext* const _mark_context;
680 bool _evac_in_progress;
681 Thread* const _thread;
682
683 public:
684 ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
685 void do_oop(oop* p);
686 void do_oop(narrowOop* p);
687 };
688
689 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
690 _heap(ShenandoahHeap::heap()),
691 _mark_context(ShenandoahHeap::heap()->marking_context()),
692 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
693 _thread(Thread::current()) {
694 }
695
696 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
697 const oop obj = RawAccess<>::oop_load(p);
698 if (!CompressedOops::is_null(obj)) {
699 if (!_mark_context->is_marked(obj)) {
700 shenandoah_assert_correct(p, obj);
701 ShenandoahHeap::atomic_clear_oop(p, obj);
702 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
703 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
704 if (resolved == obj) {
705 resolved = _heap->evacuate_object(obj, _thread);
706 }
707 ShenandoahHeap::atomic_update_oop(resolved, p, obj);
708 assert(_heap->cancelled_gc() ||
709 _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
710 "Sanity");
711 }
712 }
713 }
714
715 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
716 ShouldNotReachHere();
717 }
718
719 class ShenandoahIsCLDAliveClosure : public CLDClosure {
720 public:
721 void do_cld(ClassLoaderData* cld) {
722 cld->is_alive();
723 }
724 };
725
726 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
727 public:
728 void do_nmethod(nmethod* n) {
729 n->is_unloading();
730 }
731 };
732
733 // This task not only evacuates/updates marked weak roots, but also "NULL"
734 // dead weak roots.
735 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
736 private:
737 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
738
739 // Roots related to concurrent class unloading
740 ShenandoahClassLoaderDataRoots<true /* concurrent */>
741 _cld_roots;
742 ShenandoahConcurrentNMethodIterator _nmethod_itr;
743 ShenandoahPhaseTimings::Phase _phase;
744
745 public:
746 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
747 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
748 _vm_roots(phase),
749 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
750 _nmethod_itr(ShenandoahCodeRoots::table()),
751 _phase(phase) {
752 if (ShenandoahHeap::heap()->unload_classes()) {
753 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
754 _nmethod_itr.nmethods_do_begin();
755 }
756 }
757
758 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
759 if (ShenandoahHeap::heap()->unload_classes()) {
760 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
761 _nmethod_itr.nmethods_do_end();
762 }
763 // Notify runtime data structures of potentially dead oops
764 _vm_roots.report_num_dead();
765 }
766
767 void work(uint worker_id) {
768 ShenandoahConcurrentWorkerSession worker_session(worker_id);
769 {
770 ShenandoahEvacOOMScope oom;
771 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
772 // may race against OopStorage::release() calls.
773 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
774 _vm_roots.oops_do(&cl, worker_id);
775 }
776
777 // If we are going to perform concurrent class unloading later on, we need to
778 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
779 // can cleanup immediate garbage sooner.
780 if (ShenandoahHeap::heap()->unload_classes()) {
781 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
782 // CLD's holder or evacuate it.
783 {
784 ShenandoahIsCLDAliveClosure is_cld_alive;
785 _cld_roots.cld_do(&is_cld_alive, worker_id);
786 }
787
788 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
789 // The closure calls nmethod->is_unloading(). The is_unloading
790 // state is cached, therefore, during concurrent class unloading phase,
791 // we will not touch the metadata of unloading nmethods
792 {
793 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
794 ShenandoahIsNMethodAliveClosure is_nmethod_alive;
795 _nmethod_itr.nmethods_do(&is_nmethod_alive);
796 }
797 }
798 }
799 };
800
801 void ShenandoahConcurrentGC::op_weak_roots() {
802 ShenandoahHeap* const heap = ShenandoahHeap::heap();
803 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
804 // Concurrent weak root processing
805 {
806 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
807 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
808 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
809 heap->workers()->run_task(&task);
810 }
811
812 // Perform handshake to flush out dead oops
813 {
814 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
815 heap->rendezvous_threads();
816 }
817 }
818
819 void ShenandoahConcurrentGC::op_class_unloading() {
820 ShenandoahHeap* const heap = ShenandoahHeap::heap();
821 assert (heap->is_concurrent_weak_root_in_progress() &&
822 heap->unload_classes(),
823 "Checked by caller");
824 heap->do_class_unloading();
825 }
826
827 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
828 private:
829 BarrierSetNMethod* const _bs;
830 ShenandoahEvacuateUpdateMetadataClosure<> _cl;
831
832 public:
833 ShenandoahEvacUpdateCodeCacheClosure() :
834 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
835 _cl() {
836 }
837
838 void do_nmethod(nmethod* n) {
839 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
840 ShenandoahReentrantLocker locker(data->lock());
841 // Setup EvacOOM scope below reentrant lock to avoid deadlock with
842 // nmethod_entry_barrier
843 ShenandoahEvacOOMScope oom;
844 data->oops_do(&_cl, true/*fix relocation*/);
845 _bs->disarm(n);
846 }
847 };
848
849 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
850 private:
851 ShenandoahPhaseTimings::Phase _phase;
852 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
853 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
854 _cld_roots;
855 ShenandoahConcurrentNMethodIterator _nmethod_itr;
856
857 public:
858 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
859 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
860 _phase(phase),
861 _vm_roots(phase),
862 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
863 _nmethod_itr(ShenandoahCodeRoots::table()) {
864 if (!ShenandoahHeap::heap()->unload_classes()) {
865 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
866 _nmethod_itr.nmethods_do_begin();
867 }
868 }
869
870 ~ShenandoahConcurrentRootsEvacUpdateTask() {
871 if (!ShenandoahHeap::heap()->unload_classes()) {
872 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
873 _nmethod_itr.nmethods_do_end();
874 }
875 }
876
877 void work(uint worker_id) {
878 ShenandoahConcurrentWorkerSession worker_session(worker_id);
879 {
880 ShenandoahEvacOOMScope oom;
881 {
882 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
883 // may race against OopStorage::release() calls.
884 ShenandoahContextEvacuateUpdateRootsClosure cl;
885 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
886 }
887
888 {
889 ShenandoahEvacuateUpdateMetadataClosure<> cl;
890 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
891 _cld_roots.cld_do(&clds, worker_id);
892 }
893 }
894
895 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
896 if (!ShenandoahHeap::heap()->unload_classes()) {
897 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
898 ShenandoahEvacUpdateCodeCacheClosure cl;
899 _nmethod_itr.nmethods_do(&cl);
900 }
901 }
902 };
903
904 void ShenandoahConcurrentGC::op_strong_roots() {
905 ShenandoahHeap* const heap = ShenandoahHeap::heap();
906 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
907 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
908 heap->workers()->run_task(&task);
909 heap->set_concurrent_strong_root_in_progress(false);
910 }
911
912 void ShenandoahConcurrentGC::op_cleanup_early() {
913 ShenandoahHeap::heap()->free_set()->recycle_trash();
914 }
915
916 void ShenandoahConcurrentGC::op_evacuate() {
917 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
918 }
919
920 void ShenandoahConcurrentGC::op_init_updaterefs() {
921 ShenandoahHeap* const heap = ShenandoahHeap::heap();
922 heap->set_evacuation_in_progress(false);
923 heap->set_concurrent_weak_root_in_progress(false);
924 heap->prepare_update_heap_references(true /*concurrent*/);
925 heap->set_update_refs_in_progress(true);
926
927 if (ShenandoahPacing) {
928 heap->pacer()->setup_for_updaterefs();
929 }
930 }
931
932 void ShenandoahConcurrentGC::op_updaterefs() {
933 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
934 }
935
936 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
937 private:
938 ShenandoahUpdateRefsClosure _cl;
939 public:
940 ShenandoahUpdateThreadClosure();
941 void do_thread(Thread* thread);
942 };
943
944 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
945 HandshakeClosure("Shenandoah Update Thread Roots") {
946 }
947
948 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
949 if (thread->is_Java_thread()) {
950 JavaThread* jt = JavaThread::cast(thread);
951 ResourceMark rm;
952 jt->oops_do(&_cl, NULL);
953 }
954 }
955
956 void ShenandoahConcurrentGC::op_update_thread_roots() {
957 ShenandoahUpdateThreadClosure cl;
958 Handshake::execute(&cl);
959 }
960
961 void ShenandoahConcurrentGC::op_final_updaterefs() {
962 ShenandoahHeap* const heap = ShenandoahHeap::heap();
963 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
964 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
965
966 heap->finish_concurrent_roots();
967
968 // Clear cancelled GC, if set. On cancellation path, the block before would handle
969 // everything.
970 if (heap->cancelled_gc()) {
971 heap->clear_cancelled_gc();
972 }
973
974 // Has to be done before cset is clear
975 if (ShenandoahVerify) {
976 heap->verifier()->verify_roots_in_to_space();
977 }
978
979 heap->update_heap_region_states(true /*concurrent*/);
980
981 heap->set_update_refs_in_progress(false);
982 heap->set_has_forwarded_objects(false);
983
984 if (ShenandoahVerify) {
985 heap->verifier()->verify_after_updaterefs();
986 }
987
988 if (VerifyAfterGC) {
989 Universe::verify();
990 }
991
992 heap->rebuild_free_set(true /*concurrent*/);
993 }
994
995 void ShenandoahConcurrentGC::op_final_roots() {
996 ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
997 }
998
999 void ShenandoahConcurrentGC::op_cleanup_complete() {
1000 ShenandoahHeap::heap()->free_set()->recycle_trash();
1001 }
1002
1003 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1004 if (ShenandoahHeap::heap()->cancelled_gc()) {
1005 _degen_point = point;
1006 return true;
1007 }
1008 return false;
1009 }
1010
1011 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1012 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1013 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1014 if (heap->unload_classes()) {
1015 return "Pause Init Mark (unload classes)";
1016 } else {
1017 return "Pause Init Mark";
1018 }
1019 }
1020
1021 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1022 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1023 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1024 if (heap->unload_classes()) {
1025 return "Pause Final Mark (unload classes)";
1026 } else {
1027 return "Pause Final Mark";
1028 }
1029 }
1030
1031 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1032 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1033 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1034 if (heap->unload_classes()) {
1035 return "Concurrent marking (unload classes)";
1036 } else {
1037 return "Concurrent marking";
1038 }
1039 }