1 /*
2 * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/continuationGCSupport.hpp"
30 #include "gc/shared/fullGCForwarding.inline.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/preservedMarks.inline.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "gc/shared/workerThread.hpp"
35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
36 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
37 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
38 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
39 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
40 #include "gc/shenandoah/shenandoahFreeSet.hpp"
41 #include "gc/shenandoah/shenandoahFullGC.hpp"
42 #include "gc/shenandoah/shenandoahGenerationalFullGC.hpp"
43 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
48 #include "gc/shenandoah/shenandoahMark.inline.hpp"
49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
50 #include "gc/shenandoah/shenandoahMetrics.hpp"
51 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
52 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
53 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
54 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
55 #include "gc/shenandoah/shenandoahSTWMark.hpp"
56 #include "gc/shenandoah/shenandoahUtils.hpp"
57 #include "gc/shenandoah/shenandoahVerifier.hpp"
58 #include "gc/shenandoah/shenandoahVMOperations.hpp"
59 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
60 #include "memory/metaspaceUtils.hpp"
61 #include "memory/universe.hpp"
62 #include "oops/compressedOops.inline.hpp"
63 #include "oops/oop.inline.hpp"
64 #include "runtime/orderAccess.hpp"
65 #include "runtime/vmThread.hpp"
66 #include "utilities/copy.hpp"
67 #include "utilities/events.hpp"
68 #include "utilities/growableArray.hpp"
69
70 ShenandoahFullGC::ShenandoahFullGC() :
71 ShenandoahGC(ShenandoahHeap::heap()->global_generation()),
72 _gc_timer(ShenandoahHeap::heap()->gc_timer()),
73 _preserved_marks(new PreservedMarksSet(true)) {}
74
75 ShenandoahFullGC::~ShenandoahFullGC() {
76 delete _preserved_marks;
77 }
78
79 bool ShenandoahFullGC::collect(GCCause::Cause cause) {
80 vmop_entry_full(cause);
81 // Always success
82 return true;
83 }
84
85 void ShenandoahFullGC::vmop_entry_full(GCCause::Cause cause) {
86 ShenandoahHeap* const heap = ShenandoahHeap::heap();
87 TraceCollectorStats tcs(heap->monitoring_support()->full_stw_collection_counters());
88 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
89
90 heap->try_inject_alloc_failure();
91 VM_ShenandoahFullGC op(cause, this);
92 VMThread::execute(&op);
93 }
94
95 void ShenandoahFullGC::entry_full(GCCause::Cause cause) {
96 static const char* msg = "Pause Full";
97 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
98 EventMark em("%s", msg);
99
100 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
101 ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
102 "full gc");
103
104 op_full(cause);
105 }
106
107 void ShenandoahFullGC::op_full(GCCause::Cause cause) {
108 ShenandoahHeap* const heap = ShenandoahHeap::heap();
109
110 ShenandoahMetricsSnapshot metrics(heap->free_set());
111
112 // Perform full GC
113 do_it(cause);
114
115 if (heap->mode()->is_generational()) {
116 ShenandoahGenerationalFullGC::handle_completion(heap);
117 }
118
119 if (metrics.is_good_progress()) {
120 heap->notify_gc_progress();
121 } else {
122 // Nothing to do. Tell the allocation path that we have failed to make
123 // progress, and it can finally fail.
124 heap->notify_gc_no_progress();
125 }
126
127 // Regardless if progress was made, we record that we completed a "successful" full GC.
128 _generation->heuristics()->record_success_full();
129 heap->shenandoah_policy()->record_success_full();
130
131 {
132 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_propagate_gc_state);
133 heap->propagate_gc_state_to_all_threads();
134 }
135 }
136
137 void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
138 ShenandoahHeap* heap = ShenandoahHeap::heap();
139 heap->release_injected_pins();
140
141 // A full GC may be entered directly, or as an upgrade from a failed
142 // degenerated GC. In the latter case, self-forwarded objects may be
143 // present from the failed evacuation. Drain those marks before any phase
144 // (verify, update_roots, phase1_mark_heap) walks headers.
145 {
146 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_un_self_forward);
147 heap->un_self_forward_cset_regions();
148 }
149
150 if (heap->mode()->is_generational()) {
151 ShenandoahGenerationalFullGC::prepare();
152 }
153
154 if (ShenandoahVerify) {
155 heap->verifier()->verify_before_fullgc(_generation);
156 }
157
158 if (VerifyBeforeGC) {
159 Universe::verify();
160 }
161
162 // Degenerated GC may carry concurrent root flags when upgrading to
163 // full GC. We need to reset it before mutators resume.
164 heap->set_concurrent_strong_root_in_progress(false);
165 heap->set_concurrent_weak_root_in_progress(false);
166
167 heap->set_full_gc_in_progress(true);
168
169 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
170 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
171
172 {
173 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_pre);
174 heap->pre_full_gc_dump(_gc_timer);
175 }
176
177 {
178 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
179 // Full GC is supposed to recover from any GC state:
180
181 // a0. Remember if we have forwarded objects
182 bool has_forwarded_objects = heap->has_forwarded_objects();
183
184 // a1. Cancel evacuation, if in progress
185 if (heap->is_evacuation_in_progress()) {
186 heap->set_evacuation_in_progress(false);
187 }
188 assert(!heap->is_evacuation_in_progress(), "sanity");
189
190 // a2. Cancel update-refs, if in progress
191 if (heap->is_update_refs_in_progress()) {
192 heap->set_update_refs_in_progress(false);
193 }
194 assert(!heap->is_update_refs_in_progress(), "sanity");
195
196 // b. Cancel all concurrent marks, if in progress
197 if (heap->is_concurrent_mark_in_progress()) {
198 heap->cancel_concurrent_mark();
199 }
200 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
201
202 // c. Update roots if this full GC is due to evac-oom, which may carry from-space pointers in roots.
203 if (has_forwarded_objects) {
204 update_roots(true /*full_gc*/);
205 }
206
207 // d. Abandon reference discovery and clear all discovered references.
208 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
209 rp->abandon_partial_discovery();
210
211 // e. Sync pinned region status from the CP marks
212 heap->sync_pinned_region_status();
213
214 if (heap->mode()->is_generational()) {
215 ShenandoahGenerationalFullGC::restore_top_before_promote(heap);
216 }
217
218 // The rest of prologue:
219 _preserved_marks->init(heap->workers()->active_workers());
220
221 assert(heap->has_forwarded_objects() == has_forwarded_objects, "This should not change");
222 }
223
224 if (UseTLAB) {
225 // Note: PLABs are also retired with GCLABs in generational mode.
226 heap->gclabs_retire(ResizeTLAB);
227 heap->tlabs_retire(ResizeTLAB);
228 }
229
230 OrderAccess::fence();
231
232 phase1_mark_heap();
233
234 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
235 // Coming out of Full GC, we would not have any forwarded objects.
236 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
237 heap->set_has_forwarded_objects(false);
238
239 heap->set_full_gc_move_in_progress(true);
240
241 // Setup workers for the rest
242 OrderAccess::fence();
243
244 // Initialize worker slices
245 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
246 for (uint i = 0; i < heap->max_workers(); i++) {
247 worker_slices[i] = new ShenandoahHeapRegionSet();
248 }
249
250 {
251 // The rest of code performs region moves, where region status is undefined
252 // until all phases run together.
253 ShenandoahHeapLocker lock(heap->lock());
254
255 phase2_calculate_target_addresses(worker_slices);
256
257 OrderAccess::fence();
258
259 phase3_update_references();
260
261 phase4_compact_objects(worker_slices);
262
263 phase5_epilog();
264 }
265 heap->start_idle_span();
266
267 // Resize metaspace
268 MetaspaceGC::compute_new_size();
269
270 // Free worker slices
271 for (uint i = 0; i < heap->max_workers(); i++) {
272 delete worker_slices[i];
273 }
274 FREE_C_HEAP_ARRAY(worker_slices);
275
276 heap->set_full_gc_move_in_progress(false);
277 heap->set_full_gc_in_progress(false);
278
279 DEBUG_ONLY(heap->assert_no_self_forwards());
280
281 if (ShenandoahVerify) {
282 heap->verifier()->verify_after_fullgc(_generation);
283 }
284
285 if (VerifyAfterGC) {
286 Universe::verify();
287 }
288
289 {
290 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_post);
291 heap->post_full_gc_dump(_gc_timer);
292 }
293 }
294
295 void ShenandoahFullGC::phase1_mark_heap() {
296 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
297 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
298
299 ShenandoahHeap* heap = ShenandoahHeap::heap();
300
301 _generation->reset_mark_bitmap<true, true>();
302 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
303 assert(!_generation->is_mark_complete(), "sanity");
304
305 heap->set_unload_classes(_generation->heuristics()->can_unload_classes());
306
307 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
308 // enable ("weak") refs discovery
309 rp->set_soft_reference_policy(true); // forcefully purge all soft references
310
311 ShenandoahSTWMark mark(_generation, true /*full_gc*/);
312 mark.mark();
313 heap->parallel_cleaning(_generation, true /* full_gc */);
314
315 if (ShenandoahHeap::heap()->mode()->is_generational()) {
316 ShenandoahGenerationalFullGC::log_live_in_old(heap);
317 }
318 }
319
320 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
321 private:
322 PreservedMarks* const _preserved_marks;
323 ShenandoahHeap* const _heap;
324 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
325 int _empty_regions_pos;
326 ShenandoahHeapRegion* _to_region;
327 ShenandoahHeapRegion* _from_region;
328 HeapWord* _compact_point;
329
330 public:
331 ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
332 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
333 ShenandoahHeapRegion* to_region) :
334 _preserved_marks(preserved_marks),
335 _heap(ShenandoahHeap::heap()),
336 _empty_regions(empty_regions),
337 _empty_regions_pos(0),
338 _to_region(to_region),
339 _from_region(nullptr),
340 _compact_point(to_region->bottom()) {}
341
342 void set_from_region(ShenandoahHeapRegion* from_region) {
343 _from_region = from_region;
344 }
345
346 void finish() {
347 assert(_to_region != nullptr, "should not happen");
348 _to_region->set_new_top(_compact_point);
349 }
350
351 bool is_compact_same_region() {
352 return _from_region == _to_region;
353 }
354
355 int empty_regions_pos() {
356 return _empty_regions_pos;
357 }
358
359 void do_object(oop p) override {
360 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
361 assert(_from_region != nullptr, "must set before work");
362 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
363 assert(_heap->marking_context()->is_marked(p), "must be marked");
364 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
365
366 size_t obj_size = p->size();
367 if (_compact_point + obj_size > _to_region->end()) {
368 finish();
369
370 // Object doesn't fit. Pick next empty region and start compacting there.
371 ShenandoahHeapRegion* new_to_region;
372 if (_empty_regions_pos < _empty_regions.length()) {
373 new_to_region = _empty_regions.at(_empty_regions_pos);
374 _empty_regions_pos++;
375 } else {
376 // Out of empty region? Compact within the same region.
377 new_to_region = _from_region;
378 }
379
380 assert(new_to_region != _to_region, "must not reuse same to-region");
381 assert(new_to_region != nullptr, "must not be null");
382 _to_region = new_to_region;
383 _compact_point = _to_region->bottom();
384 }
385
386 // Object fits into current region, record new location, if object does not move:
387 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
388 shenandoah_assert_not_forwarded(nullptr, p);
389 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
390 _preserved_marks->push_if_necessary(p, p->mark());
391 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
392 }
393 _compact_point += obj_size;
394 }
395 };
396
397 class ShenandoahPrepareForCompactionTask : public WorkerTask {
398 private:
399 PreservedMarksSet* const _preserved_marks;
400 ShenandoahHeap* const _heap;
401 ShenandoahHeapRegionSet** const _worker_slices;
402
403 public:
404 ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
405 WorkerTask("Shenandoah Prepare For Compaction"),
406 _preserved_marks(preserved_marks),
407 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
408 }
409
410 static bool is_candidate_region(ShenandoahHeapRegion* r) {
411 // Empty region: get it into the slice to defragment the slice itself.
412 // We could have skipped this without violating correctness, but we really
413 // want to compact all live regions to the start of the heap, which sometimes
414 // means moving them into the fully empty regions.
415 if (r->is_empty()) return true;
416
417 // Can move the region, and this is not the humongous region. Humongous
418 // moves are special cased here, because their moves are handled separately.
419 return r->is_stw_move_allowed() && !r->is_humongous();
420 }
421
422 void work(uint worker_id) override;
423 private:
424 template<typename ClosureType>
425 void prepare_for_compaction(ClosureType& cl,
426 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
427 ShenandoahHeapRegionSetIterator& it,
428 ShenandoahHeapRegion* from_region);
429 };
430
431 void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
432 ShenandoahParallelWorkerSession worker_session(worker_id);
433 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
434 ShenandoahHeapRegionSetIterator it(slice);
435 ShenandoahHeapRegion* from_region = it.next();
436 // No work?
437 if (from_region == nullptr) {
438 return;
439 }
440
441 // Sliding compaction. Walk all regions in the slice, and compact them.
442 // Remember empty regions and reuse them as needed.
443 ResourceMark rm;
444
445 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
446
447 if (_heap->mode()->is_generational()) {
448 ShenandoahPrepareForGenerationalCompactionObjectClosure cl(_preserved_marks->get(worker_id),
449 empty_regions, from_region, worker_id);
450 prepare_for_compaction(cl, empty_regions, it, from_region);
451 } else {
452 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
453 prepare_for_compaction(cl, empty_regions, it, from_region);
454 }
455 }
456
457 template<typename ClosureType>
458 void ShenandoahPrepareForCompactionTask::prepare_for_compaction(ClosureType& cl,
459 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
460 ShenandoahHeapRegionSetIterator& it,
461 ShenandoahHeapRegion* from_region) {
462 while (from_region != nullptr) {
463 assert(is_candidate_region(from_region), "Sanity");
464 cl.set_from_region(from_region);
465 if (from_region->has_live()) {
466 _heap->marked_object_iterate(from_region, &cl);
467 }
468
469 // Compacted the region to somewhere else? From-region is empty then.
470 if (!cl.is_compact_same_region()) {
471 empty_regions.append(from_region);
472 }
473 from_region = it.next();
474 }
475 cl.finish();
476
477 // Mark all remaining regions as empty
478 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
479 ShenandoahHeapRegion* r = empty_regions.at(pos);
480 r->set_new_top(r->bottom());
481 }
482 }
483
484 void ShenandoahFullGC::calculate_target_humongous_objects() {
485 ShenandoahHeap* heap = ShenandoahHeap::heap();
486
487 // Compute the new addresses for humongous objects. We need to do this after addresses
488 // for regular objects are calculated, and we know what regions in heap suffix are
489 // available for humongous moves.
490 //
491 // Scan the heap backwards, because we are compacting humongous regions towards the end.
492 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
493 // humongous start there.
494 //
495 // The complication is potential non-movable regions during the scan. If such region is
496 // detected, then sliding restarts towards that non-movable region.
497
498 size_t to_begin = heap->num_regions();
499 size_t to_end = heap->num_regions();
500
501 log_debug(gc)("Full GC calculating target humongous objects from end %zu", to_end);
502 for (size_t c = heap->num_regions(); c > 0; c--) {
503 ShenandoahHeapRegion *r = heap->get_region(c - 1);
504 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
505 // To-region candidate: record this, and continue scan
506 to_begin = r->index();
507 continue;
508 }
509
510 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
511 // From-region candidate: movable humongous region
512 oop old_obj = cast_to_oop(r->bottom());
513 size_t words_size = old_obj->size();
514 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
515
516 size_t start = to_end - num_regions;
517
518 if (start >= to_begin && start != r->index()) {
519 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
520 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
521 FullGCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
522 to_end = start;
523 continue;
524 }
525 }
526
527 // Failed to fit. Scan starting from current region.
528 to_begin = r->index();
529 to_end = r->index();
530 }
531 }
532
533 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
534 public:
535 void heap_region_do(ShenandoahHeapRegion* r) override {
536 if (r->is_trash()) {
537 r->try_recycle_under_lock();
538 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
539 }
540 if (r->is_cset()) {
541 // Leave affiliation unchanged
542 r->make_regular_bypass();
543 }
544 if (r->is_empty_uncommitted()) {
545 r->make_committed_bypass();
546 }
547 assert (r->is_committed(), "only committed regions in heap now, see region %zu", r->index());
548
549 // Record current region occupancy: this communicates empty regions are free
550 // to the rest of Full GC code.
551 r->set_new_top(r->top());
552 }
553 };
554
555 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
556 private:
557 ShenandoahHeap* const _heap;
558 ShenandoahMarkingContext* const _ctx;
559
560 public:
561 ShenandoahTrashImmediateGarbageClosure() :
562 _heap(ShenandoahHeap::heap()),
563 _ctx(ShenandoahHeap::heap()->global_generation()->complete_marking_context()) {}
564
565 void heap_region_do(ShenandoahHeapRegion* r) override {
566 if (r->is_humongous_start()) {
567 oop humongous_obj = cast_to_oop(r->bottom());
568 if (!_ctx->is_marked(humongous_obj)) {
569 assert(!r->has_live(), "Region %zu is not marked, should not have live", r->index());
570 _heap->trash_humongous_region_at(r);
571 } else {
572 assert(r->has_live(), "Region %zu should have live", r->index());
573 }
574 } else if (r->is_humongous_continuation()) {
575 // If we hit continuation, the non-live humongous starts should have been trashed already
576 assert(r->humongous_start_region()->has_live(), "Region %zu should have live", r->index());
577 } else if (r->is_regular()) {
578 if (!r->has_live()) {
579 r->make_trash_immediate();
580 }
581 }
582 }
583 };
584
585 void ShenandoahFullGC::distribute_slices(ShenandoahHeapRegionSet** worker_slices) {
586 ShenandoahHeap* heap = ShenandoahHeap::heap();
587
588 uint n_workers = heap->workers()->active_workers();
589 size_t n_regions = heap->num_regions();
590
591 // What we want to accomplish: have the dense prefix of data, while still balancing
592 // out the parallel work.
593 //
594 // Assuming the amount of work is driven by the live data that needs moving, we can slice
595 // the entire heap into equal-live-sized prefix slices, and compact into them. So, each
596 // thread takes all regions in its prefix subset, and then it takes some regions from
597 // the tail.
598 //
599 // Tail region selection becomes interesting.
600 //
601 // First, we want to distribute the regions fairly between the workers, and those regions
602 // might have different amount of live data. So, until we sure no workers need live data,
603 // we need to only take what the worker needs.
604 //
605 // Second, since we slide everything to the left in each slice, the most busy regions
606 // would be the ones on the left. Which means we want to have all workers have their after-tail
607 // regions as close to the left as possible.
608 //
609 // The easiest way to do this is to distribute after-tail regions in round-robin between
610 // workers that still need live data.
611 //
612 // Consider parallel workers A, B, C, then the target slice layout would be:
613 //
614 // AAAAAAAABBBBBBBBCCCCCCCC|ABCABCABCABCABCABCABCABABABABABABABABABABAAAAA
615 //
616 // (.....dense-prefix.....) (.....................tail...................)
617 // [all regions fully live] [left-most regions are fuller that right-most]
618 //
619
620 // Compute how much live data is there. This would approximate the size of dense prefix
621 // we target to create.
622 size_t total_live = 0;
623 for (size_t idx = 0; idx < n_regions; idx++) {
624 ShenandoahHeapRegion *r = heap->get_region(idx);
625 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
626 total_live += r->get_live_data_words();
627 }
628 }
629
630 // Estimate the size for the dense prefix. Note that we specifically count only the
631 // "full" regions, so there would be some non-full regions in the slice tail.
632 size_t live_per_worker = total_live / n_workers;
633 size_t prefix_regions_per_worker = live_per_worker / ShenandoahHeapRegion::region_size_words();
634 size_t prefix_regions_total = prefix_regions_per_worker * n_workers;
635 prefix_regions_total = MIN2(prefix_regions_total, n_regions);
636 assert(prefix_regions_total <= n_regions, "Sanity");
637
638 // There might be non-candidate regions in the prefix. To compute where the tail actually
639 // ends up being, we need to account those as well.
640 size_t prefix_end = prefix_regions_total;
641 for (size_t idx = 0; idx < prefix_regions_total; idx++) {
642 ShenandoahHeapRegion *r = heap->get_region(idx);
643 if (!ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
644 prefix_end++;
645 }
646 }
647 prefix_end = MIN2(prefix_end, n_regions);
648 assert(prefix_end <= n_regions, "Sanity");
649
650 // Distribute prefix regions per worker: each thread definitely gets its own same-sized
651 // subset of dense prefix.
652 size_t prefix_idx = 0;
653
654 size_t* live = NEW_C_HEAP_ARRAY(size_t, n_workers, mtGC);
655
656 for (size_t wid = 0; wid < n_workers; wid++) {
657 ShenandoahHeapRegionSet* slice = worker_slices[wid];
658
659 live[wid] = 0;
660 size_t regs = 0;
661
662 // Add all prefix regions for this worker
663 while (prefix_idx < prefix_end && regs < prefix_regions_per_worker) {
664 ShenandoahHeapRegion *r = heap->get_region(prefix_idx);
665 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
666 slice->add_region(r);
667 live[wid] += r->get_live_data_words();
668 regs++;
669 }
670 prefix_idx++;
671 }
672 }
673
674 // Distribute the tail among workers in round-robin fashion.
675 size_t wid = n_workers - 1;
676
677 for (size_t tail_idx = prefix_end; tail_idx < n_regions; tail_idx++) {
678 ShenandoahHeapRegion *r = heap->get_region(tail_idx);
679 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
680 assert(wid < n_workers, "Sanity");
681
682 size_t live_region = r->get_live_data_words();
683
684 // Select next worker that still needs live data.
685 size_t old_wid = wid;
686 do {
687 wid++;
688 if (wid == n_workers) wid = 0;
689 } while (live[wid] + live_region >= live_per_worker && old_wid != wid);
690
691 if (old_wid == wid) {
692 // Circled back to the same worker? This means liveness data was
693 // miscalculated. Bump the live_per_worker limit so that
694 // everyone gets a piece of the leftover work.
695 live_per_worker += ShenandoahHeapRegion::region_size_words();
696 }
697
698 worker_slices[wid]->add_region(r);
699 live[wid] += live_region;
700 }
701 }
702
703 FREE_C_HEAP_ARRAY(live);
704
705 #ifdef ASSERT
706 ResourceBitMap map(n_regions);
707 for (size_t wid = 0; wid < n_workers; wid++) {
708 ShenandoahHeapRegionSetIterator it(worker_slices[wid]);
709 ShenandoahHeapRegion* r = it.next();
710 while (r != nullptr) {
711 size_t idx = r->index();
712 assert(ShenandoahPrepareForCompactionTask::is_candidate_region(r), "Sanity: %zu", idx);
713 assert(!map.at(idx), "No region distributed twice: %zu", idx);
714 map.at_put(idx, true);
715 r = it.next();
716 }
717 }
718
719 for (size_t rid = 0; rid < n_regions; rid++) {
720 bool is_candidate = ShenandoahPrepareForCompactionTask::is_candidate_region(heap->get_region(rid));
721 bool is_distributed = map.at(rid);
722 assert(is_distributed || !is_candidate, "All candidates are distributed: %zu", rid);
723 }
724 #endif
725 }
726
727 void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
728 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
729 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
730
731 ShenandoahHeap* heap = ShenandoahHeap::heap();
732
733 // About to figure out which regions can be compacted, make sure pinning status
734 // had been updated in GC prologue.
735 heap->assert_pinned_region_status();
736
737 {
738 // Trash the immediately collectible regions before computing addresses
739 ShenandoahTrashImmediateGarbageClosure trash_immediate_garbage;
740 ShenandoahExcludeRegionClosure<FREE> cl(&trash_immediate_garbage);
741 heap->heap_region_iterate(&cl);
742
743 // Make sure regions are in good state: committed, active, clean.
744 // This is needed because we are potentially sliding the data through them.
745 ShenandoahEnsureHeapActiveClosure ecl;
746 heap->heap_region_iterate(&ecl);
747 }
748
749 // Compute the new addresses for regular objects
750 {
751 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
752
753 distribute_slices(worker_slices);
754
755 ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices);
756 heap->workers()->run_task(&task);
757 }
758
759 // Compute the new addresses for humongous objects
760 {
761 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
762 calculate_target_humongous_objects();
763 }
764 }
765
766 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
767 private:
768 ShenandoahMarkingContext* const _ctx;
769
770 template <class T>
771 inline void do_oop_work(T* p) {
772 T o = RawAccess<>::oop_load(p);
773 if (!CompressedOops::is_null(o)) {
774 oop obj = CompressedOops::decode_not_null(o);
775 assert(_ctx->is_marked(obj), "must be marked");
776 if (FullGCForwarding::is_forwarded(obj)) {
777 oop forw = FullGCForwarding::forwardee(obj);
778 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
779 }
780 }
781 }
782
783 public:
784 ShenandoahAdjustPointersClosure() :
785 _ctx(ShenandoahHeap::heap()->global_generation()->complete_marking_context()) {}
786
787 void do_oop(oop* p) { do_oop_work(p); }
788 void do_oop(narrowOop* p) { do_oop_work(p); }
789 void do_method(Method* m) {}
790 void do_nmethod(nmethod* nm) {}
791 };
792
793 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
794 private:
795 ShenandoahAdjustPointersClosure _cl;
796
797 public:
798 void do_object(oop p) override {
799 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be complete");
800 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
801 p->oop_iterate(&_cl);
802 }
803 };
804
805 class ShenandoahAdjustPointersTask : public WorkerTask {
806 private:
807 ShenandoahHeap* const _heap;
808 ShenandoahRegionIterator _regions;
809
810 public:
811 ShenandoahAdjustPointersTask() :
812 WorkerTask("Shenandoah Adjust Pointers"),
813 _heap(ShenandoahHeap::heap()) {
814 }
815
816 void work(uint worker_id) override {
817 ShenandoahParallelWorkerSession worker_session(worker_id);
818 ShenandoahAdjustPointersObjectClosure obj_cl;
819 ShenandoahHeapRegion* r = _regions.next();
820 while (r != nullptr) {
821 if (!r->is_humongous_continuation() && r->has_live()) {
822 _heap->marked_object_iterate(r, &obj_cl);
823 }
824 if (_heap->mode()->is_generational()) {
825 ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(r);
826 }
827 r = _regions.next();
828 }
829 }
830 };
831
832 class ShenandoahAdjustRootPointersTask : public WorkerTask {
833 private:
834 ShenandoahRootAdjuster* _rp;
835 PreservedMarksSet* _preserved_marks;
836 public:
837 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
838 WorkerTask("Shenandoah Adjust Root Pointers"),
839 _rp(rp),
840 _preserved_marks(preserved_marks) {}
841
842 void work(uint worker_id) override {
843 ShenandoahParallelWorkerSession worker_session(worker_id);
844 ShenandoahAdjustPointersClosure cl;
845 _rp->roots_do(worker_id, &cl);
846 _preserved_marks->get(worker_id)->adjust_during_full_gc();
847 }
848 };
849
850 void ShenandoahFullGC::phase3_update_references() {
851 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
852 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
853
854 ShenandoahHeap* heap = ShenandoahHeap::heap();
855
856 WorkerThreads* workers = heap->workers();
857 uint nworkers = workers->active_workers();
858 {
859 #ifdef COMPILER2
860 DerivedPointerTable::clear();
861 #endif // COMPILER2
862 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_adjust_roots);
863 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
864 workers->run_task(&task);
865 #ifdef COMPILER2
866 DerivedPointerTable::update_pointers();
867 #endif // COMPILER2
868 }
869
870 ShenandoahAdjustPointersTask adjust_pointers_task;
871 workers->run_task(&adjust_pointers_task);
872 }
873
874 class ShenandoahCompactObjectsClosure : public ObjectClosure {
875 private:
876 uint const _worker_id;
877
878 public:
879 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
880 _worker_id(worker_id) {}
881
882 void do_object(oop p) override {
883 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
884 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
885 size_t size = p->size();
886 if (FullGCForwarding::is_forwarded(p)) {
887 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
888 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
889 assert(compact_from != compact_to, "Forwarded object should move");
890 Copy::aligned_conjoint_words(compact_from, compact_to, size);
891 oop new_obj = cast_to_oop(compact_to);
892
893 // Restore the mark word before relativizing the stack chunk. The copy's
894 // mark word contains the full GC forwarding encoding, which would cause
895 // is_stackChunk() to read garbage (especially with compact headers).
896 new_obj->init_mark();
897 ContinuationGCSupport::relativize_stack_chunk(new_obj);
898 }
899 }
900 };
901
902 class ShenandoahCompactObjectsTask : public WorkerTask {
903 private:
904 ShenandoahHeap* const _heap;
905 ShenandoahHeapRegionSet** const _worker_slices;
906
907 public:
908 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
909 WorkerTask("Shenandoah Compact Objects"),
910 _heap(ShenandoahHeap::heap()),
911 _worker_slices(worker_slices) {
912 }
913
914 void work(uint worker_id) override {
915 ShenandoahParallelWorkerSession worker_session(worker_id);
916 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
917
918 ShenandoahCompactObjectsClosure cl(worker_id);
919 ShenandoahHeapRegion* r = slice.next();
920 while (r != nullptr) {
921 assert(!r->is_humongous(), "must not get humongous regions here");
922 if (r->has_live()) {
923 _heap->marked_object_iterate(r, &cl);
924 }
925 r->set_top(r->new_top());
926 r = slice.next();
927 }
928 }
929 };
930
931 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
932 private:
933 ShenandoahHeap* const _heap;
934 bool _is_generational;
935 size_t _young_regions, _young_usage, _young_humongous_waste;
936 size_t _old_regions, _old_usage, _old_humongous_waste;
937
938 public:
939 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()),
940 _is_generational(_heap->mode()->is_generational()),
941 _young_regions(0),
942 _young_usage(0),
943 _young_humongous_waste(0),
944 _old_regions(0),
945 _old_usage(0),
946 _old_humongous_waste(0)
947 {
948 _heap->free_set()->clear();
949 }
950
951 void heap_region_do(ShenandoahHeapRegion* r) override {
952 assert (!r->is_cset(), "cset regions should have been demoted already");
953
954 // Need to reset the complete-top-at-mark-start pointer here because
955 // the complete marking bitmap is no longer valid. This ensures
956 // size-based iteration in marked_object_iterate().
957 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
958 // pinned regions.
959 if (!r->is_pinned()) {
960 _heap->marking_context()->reset_top_at_mark_start(r);
961 }
962
963 size_t live = r->used();
964
965 // Make empty regions that have been allocated into regular
966 if (r->is_empty() && live > 0) {
967 if (!_is_generational) {
968 r->make_affiliated_maybe();
969 }
970 // else, generational mode compaction has already established affiliation.
971 r->make_regular_bypass();
972 if (ZapUnusedHeapArea) {
973 SpaceMangler::mangle_region(MemRegion(r->top(), r->end()));
974 }
975 }
976
977 // Reclaim regular regions that became empty
978 if (r->is_regular() && live == 0) {
979 r->make_trash();
980 }
981
982 // Recycle all trash regions
983 if (r->is_trash()) {
984 live = 0;
985 r->try_recycle_under_lock();
986 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
987 } else {
988 if (r->is_old()) {
989 ShenandoahGenerationalFullGC::account_for_region(r, _old_regions, _old_usage, _old_humongous_waste);
990 } else if (r->is_young()) {
991 ShenandoahGenerationalFullGC::account_for_region(r, _young_regions, _young_usage, _young_humongous_waste);
992 }
993 }
994 r->set_live_data(live);
995 r->reset_alloc_metadata();
996 }
997 };
998
999 void ShenandoahFullGC::compact_humongous_objects() {
1000 // Compact humongous regions, based on their fwdptr objects.
1001 //
1002 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
1003 // humongous regions are already compacted, and do not require further moves, which alleviates
1004 // sliding costs. We may consider doing this in parallel in the future.
1005
1006 ShenandoahHeap* heap = ShenandoahHeap::heap();
1007
1008 for (size_t c = heap->num_regions(); c > 0; c--) {
1009 ShenandoahHeapRegion* r = heap->get_region(c - 1);
1010 if (r->is_humongous_start()) {
1011 oop old_obj = cast_to_oop(r->bottom());
1012 if (!FullGCForwarding::is_forwarded(old_obj)) {
1013 // No need to move the object, it stays at the same slot
1014 continue;
1015 }
1016 size_t words_size = old_obj->size();
1017 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1018
1019 size_t old_start = r->index();
1020 size_t old_end = old_start + num_regions - 1;
1021 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1022 size_t new_end = new_start + num_regions - 1;
1023 assert(old_start != new_start, "must be real move");
1024 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1025
1026 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1027 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
1028 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1029
1030 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1031 new_obj->init_mark();
1032
1033 {
1034 ShenandoahAffiliation original_affiliation = r->affiliation();
1035 for (size_t c = old_start; c <= old_end; c++) {
1036 ShenandoahHeapRegion* r = heap->get_region(c);
1037 // Leave humongous region affiliation unchanged.
1038 r->make_regular_bypass();
1039 r->set_top(r->bottom());
1040 }
1041
1042 for (size_t c = new_start; c <= new_end; c++) {
1043 ShenandoahHeapRegion* r = heap->get_region(c);
1044 if (c == new_start) {
1045 r->make_humongous_start_bypass(original_affiliation);
1046 } else {
1047 r->make_humongous_cont_bypass(original_affiliation);
1048 }
1049
1050 // Trailing region may be non-full, record the remainder there
1051 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
1052 if ((c == new_end) && (remainder != 0)) {
1053 r->set_top(r->bottom() + remainder);
1054 } else {
1055 r->set_top(r->end());
1056 }
1057
1058 r->reset_alloc_metadata();
1059 }
1060 }
1061 }
1062 }
1063 }
1064
1065 // This is slightly different to ShHeap::reset_next_mark_bitmap:
1066 // we need to remain able to walk pinned regions.
1067 // Since pinned region do not move and don't get compacted, we will get holes with
1068 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
1069 // cannot be iterated over using oop->size()). The only way to safely iterate over those is using
1070 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
1071 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
1072 class ShenandoahMCResetCompleteBitmapTask : public WorkerTask {
1073 private:
1074 ShenandoahRegionIterator _regions;
1075
1076 public:
1077 ShenandoahMCResetCompleteBitmapTask() :
1078 WorkerTask("Shenandoah Reset Bitmap") {
1079 }
1080
1081 void work(uint worker_id) override {
1082 ShenandoahParallelWorkerSession worker_session(worker_id);
1083 ShenandoahHeapRegion* region = _regions.next();
1084 ShenandoahHeap* heap = ShenandoahHeap::heap();
1085 ShenandoahMarkingContext* const ctx = heap->marking_context();
1086 assert(heap->global_generation()->is_mark_complete(), "Marking must be complete");
1087 while (region != nullptr) {
1088 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
1089 ctx->clear_bitmap(region);
1090 }
1091 region = _regions.next();
1092 }
1093 }
1094 };
1095
1096 void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
1097 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
1098 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
1099
1100 ShenandoahHeap* heap = ShenandoahHeap::heap();
1101
1102 // Compact regular objects first
1103 {
1104 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
1105 ShenandoahCompactObjectsTask compact_task(worker_slices);
1106 heap->workers()->run_task(&compact_task);
1107 }
1108
1109 // Compact humongous objects after regular object moves
1110 {
1111 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
1112 compact_humongous_objects();
1113 }
1114 }
1115
1116 void ShenandoahFullGC::phase5_epilog() {
1117 GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer);
1118 ShenandoahHeap* heap = ShenandoahHeap::heap();
1119
1120 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
1121 // and must ensure the bitmap is in sync.
1122 {
1123 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
1124 ShenandoahMCResetCompleteBitmapTask task;
1125 heap->workers()->run_task(&task);
1126 }
1127
1128 // Bring regions in proper states after the collection, and set heap properties.
1129 {
1130 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
1131 ShenandoahPostCompactClosure post_compact;
1132 heap->heap_region_iterate(&post_compact);
1133 heap->collection_set()->clear();
1134 {
1135 ShenandoahFreeSet* free_set = heap->free_set();
1136 size_t young_trashed_regions, old_trashed_regions, first_old, last_old, num_old;
1137 free_set->prepare_to_rebuild(young_trashed_regions, old_trashed_regions, first_old, last_old, num_old);
1138 // We also do not expand old generation size following Full GC because we have scrambled age populations and
1139 // no longer have objects separated by age into distinct regions.
1140 if (heap->mode()->is_generational()) {
1141 ShenandoahGenerationalFullGC::compute_balances();
1142 }
1143 heap->free_set()->finish_rebuild(young_trashed_regions, old_trashed_regions, num_old);
1144 }
1145
1146 // Set mark incomplete because the marking bitmaps have been reset except pinned regions.
1147 _generation->set_mark_incomplete();
1148
1149 heap->clear_cancelled_gc();
1150 }
1151
1152 _preserved_marks->restore(heap->workers());
1153 _preserved_marks->reclaim();
1154
1155 if (heap->mode()->is_generational()) {
1156 ShenandoahGenerationalFullGC::rebuild_remembered_set(heap);
1157 }
1158 }