1 /*
2 * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/continuationGCSupport.hpp"
30 #include "gc/shared/fullGCForwarding.inline.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/preservedMarks.inline.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "gc/shared/workerThread.hpp"
35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
36 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
37 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
38 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
39 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
40 #include "gc/shenandoah/shenandoahFreeSet.hpp"
41 #include "gc/shenandoah/shenandoahFullGC.hpp"
42 #include "gc/shenandoah/shenandoahGenerationalFullGC.hpp"
43 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
48 #include "gc/shenandoah/shenandoahMark.inline.hpp"
49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
50 #include "gc/shenandoah/shenandoahMetrics.hpp"
51 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
52 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
53 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
54 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
55 #include "gc/shenandoah/shenandoahSTWMark.hpp"
56 #include "gc/shenandoah/shenandoahUtils.hpp"
57 #include "gc/shenandoah/shenandoahVerifier.hpp"
58 #include "gc/shenandoah/shenandoahVMOperations.hpp"
59 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
60 #include "memory/metaspaceUtils.hpp"
61 #include "memory/universe.hpp"
62 #include "oops/compressedOops.inline.hpp"
63 #include "oops/oop.inline.hpp"
64 #include "runtime/orderAccess.hpp"
65 #include "runtime/vmThread.hpp"
66 #include "utilities/copy.hpp"
67 #include "utilities/events.hpp"
68 #include "utilities/growableArray.hpp"
69
70 ShenandoahFullGC::ShenandoahFullGC() :
71 ShenandoahGC(ShenandoahHeap::heap()->global_generation()),
72 _gc_timer(ShenandoahHeap::heap()->gc_timer()),
73 _preserved_marks(new PreservedMarksSet(true)) {}
74
75 ShenandoahFullGC::~ShenandoahFullGC() {
76 delete _preserved_marks;
77 }
78
79 bool ShenandoahFullGC::collect(GCCause::Cause cause) {
80 vmop_entry_full(cause);
81 // Always success
82 return true;
83 }
84
85 void ShenandoahFullGC::vmop_entry_full(GCCause::Cause cause) {
86 ShenandoahHeap* const heap = ShenandoahHeap::heap();
87 TraceCollectorStats tcs(heap->monitoring_support()->full_stw_collection_counters());
88 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
89
90 heap->try_inject_alloc_failure();
91 VM_ShenandoahFullGC op(cause, this);
92 VMThread::execute(&op);
93 }
94
95 void ShenandoahFullGC::entry_full(GCCause::Cause cause) {
96 static const char* msg = "Pause Full";
97 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
98 EventMark em("%s", msg);
99
100 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
101 ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
102 "full gc");
103
104 op_full(cause);
105 }
106
107 void ShenandoahFullGC::op_full(GCCause::Cause cause) {
108 ShenandoahHeap* const heap = ShenandoahHeap::heap();
109
110 ShenandoahMetricsSnapshot metrics(heap->free_set());
111
112 // Perform full GC
113 do_it(cause);
114
115 if (heap->mode()->is_generational()) {
116 ShenandoahGenerationalFullGC::handle_completion(heap);
117 }
118
119 if (metrics.is_good_progress()) {
120 heap->notify_gc_progress();
121 } else {
122 // Nothing to do. Tell the allocation path that we have failed to make
123 // progress, and it can finally fail.
124 heap->notify_gc_no_progress();
125 }
126
127 // Regardless if progress was made, we record that we completed a "successful" full GC.
128 _generation->heuristics()->record_success_full();
129 heap->shenandoah_policy()->record_success_full();
130
131 {
132 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_propagate_gc_state);
133 heap->propagate_gc_state_to_all_threads();
134 }
135 }
136
137 void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
138 ShenandoahHeap* heap = ShenandoahHeap::heap();
139
140 if (heap->mode()->is_generational()) {
141 ShenandoahGenerationalFullGC::prepare();
142 }
143
144 if (ShenandoahVerify) {
145 heap->verifier()->verify_before_fullgc(_generation);
146 }
147
148 if (VerifyBeforeGC) {
149 Universe::verify();
150 }
151
152 // Degenerated GC may carry concurrent root flags when upgrading to
153 // full GC. We need to reset it before mutators resume.
154 heap->set_concurrent_strong_root_in_progress(false);
155 heap->set_concurrent_weak_root_in_progress(false);
156
157 heap->set_full_gc_in_progress(true);
158
159 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
160 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
161
162 {
163 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_pre);
164 heap->pre_full_gc_dump(_gc_timer);
165 }
166
167 {
168 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
169 // Full GC is supposed to recover from any GC state:
170
171 // a0. Remember if we have forwarded objects
172 bool has_forwarded_objects = heap->has_forwarded_objects();
173
174 // a1. Cancel evacuation, if in progress
175 if (heap->is_evacuation_in_progress()) {
176 heap->set_evacuation_in_progress(false);
177 }
178 assert(!heap->is_evacuation_in_progress(), "sanity");
179
180 // a2. Cancel update-refs, if in progress
181 if (heap->is_update_refs_in_progress()) {
182 heap->set_update_refs_in_progress(false);
183 }
184 assert(!heap->is_update_refs_in_progress(), "sanity");
185
186 // b. Cancel all concurrent marks, if in progress
187 if (heap->is_concurrent_mark_in_progress()) {
188 heap->cancel_concurrent_mark();
189 }
190 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
191
192 // c. Update roots if this full GC is due to evac-oom, which may carry from-space pointers in roots.
193 if (has_forwarded_objects) {
194 update_roots(true /*full_gc*/);
195 }
196
197 // d. Abandon reference discovery and clear all discovered references.
198 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
199 rp->abandon_partial_discovery();
200
201 // e. Sync pinned region status from the CP marks
202 heap->sync_pinned_region_status();
203
204 if (heap->mode()->is_generational()) {
205 ShenandoahGenerationalFullGC::restore_top_before_promote(heap);
206 }
207
208 // The rest of prologue:
209 _preserved_marks->init(heap->workers()->active_workers());
210
211 assert(heap->has_forwarded_objects() == has_forwarded_objects, "This should not change");
212 }
213
214 if (UseTLAB) {
215 // Note: PLABs are also retired with GCLABs in generational mode.
216 heap->gclabs_retire(ResizeTLAB);
217 heap->tlabs_retire(ResizeTLAB);
218 }
219
220 OrderAccess::fence();
221
222 phase1_mark_heap();
223
224 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
225 // Coming out of Full GC, we would not have any forwarded objects.
226 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
227 heap->set_has_forwarded_objects(false);
228
229 heap->set_full_gc_move_in_progress(true);
230
231 // Setup workers for the rest
232 OrderAccess::fence();
233
234 // Initialize worker slices
235 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
236 for (uint i = 0; i < heap->max_workers(); i++) {
237 worker_slices[i] = new ShenandoahHeapRegionSet();
238 }
239
240 {
241 // The rest of code performs region moves, where region status is undefined
242 // until all phases run together.
243 ShenandoahHeapLocker lock(heap->lock());
244
245 phase2_calculate_target_addresses(worker_slices);
246
247 OrderAccess::fence();
248
249 phase3_update_references();
250
251 phase4_compact_objects(worker_slices);
252
253 phase5_epilog();
254 }
255 heap->start_idle_span();
256
257 // Resize metaspace
258 MetaspaceGC::compute_new_size();
259
260 // Free worker slices
261 for (uint i = 0; i < heap->max_workers(); i++) {
262 delete worker_slices[i];
263 }
264 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
265
266 heap->set_full_gc_move_in_progress(false);
267 heap->set_full_gc_in_progress(false);
268
269 if (ShenandoahVerify) {
270 heap->verifier()->verify_after_fullgc(_generation);
271 }
272
273 if (VerifyAfterGC) {
274 Universe::verify();
275 }
276
277 {
278 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_post);
279 heap->post_full_gc_dump(_gc_timer);
280 }
281 }
282
283 void ShenandoahFullGC::phase1_mark_heap() {
284 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
285 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
286
287 ShenandoahHeap* heap = ShenandoahHeap::heap();
288
289 _generation->reset_mark_bitmap<true, true>();
290 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
291 assert(!_generation->is_mark_complete(), "sanity");
292
293 heap->set_unload_classes(_generation->heuristics()->can_unload_classes());
294
295 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
296 // enable ("weak") refs discovery
297 rp->set_soft_reference_policy(true); // forcefully purge all soft references
298
299 ShenandoahSTWMark mark(_generation, true /*full_gc*/);
300 mark.mark();
301 heap->parallel_cleaning(_generation, true /* full_gc */);
302
303 if (ShenandoahHeap::heap()->mode()->is_generational()) {
304 ShenandoahGenerationalFullGC::log_live_in_old(heap);
305 }
306 }
307
308 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
309 private:
310 PreservedMarks* const _preserved_marks;
311 ShenandoahHeap* const _heap;
312 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
313 int _empty_regions_pos;
314 ShenandoahHeapRegion* _to_region;
315 ShenandoahHeapRegion* _from_region;
316 HeapWord* _compact_point;
317
318 public:
319 ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
320 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
321 ShenandoahHeapRegion* to_region) :
322 _preserved_marks(preserved_marks),
323 _heap(ShenandoahHeap::heap()),
324 _empty_regions(empty_regions),
325 _empty_regions_pos(0),
326 _to_region(to_region),
327 _from_region(nullptr),
328 _compact_point(to_region->bottom()) {}
329
330 void set_from_region(ShenandoahHeapRegion* from_region) {
331 _from_region = from_region;
332 }
333
334 void finish() {
335 assert(_to_region != nullptr, "should not happen");
336 _to_region->set_new_top(_compact_point);
337 }
338
339 bool is_compact_same_region() {
340 return _from_region == _to_region;
341 }
342
343 int empty_regions_pos() {
344 return _empty_regions_pos;
345 }
346
347 void do_object(oop p) override {
348 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
349 assert(_from_region != nullptr, "must set before work");
350 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
351 assert(_heap->marking_context()->is_marked(p), "must be marked");
352 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
353
354 size_t obj_size = p->size();
355 if (_compact_point + obj_size > _to_region->end()) {
356 finish();
357
358 // Object doesn't fit. Pick next empty region and start compacting there.
359 ShenandoahHeapRegion* new_to_region;
360 if (_empty_regions_pos < _empty_regions.length()) {
361 new_to_region = _empty_regions.at(_empty_regions_pos);
362 _empty_regions_pos++;
363 } else {
364 // Out of empty region? Compact within the same region.
365 new_to_region = _from_region;
366 }
367
368 assert(new_to_region != _to_region, "must not reuse same to-region");
369 assert(new_to_region != nullptr, "must not be null");
370 _to_region = new_to_region;
371 _compact_point = _to_region->bottom();
372 }
373
374 // Object fits into current region, record new location, if object does not move:
375 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
376 shenandoah_assert_not_forwarded(nullptr, p);
377 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
378 _preserved_marks->push_if_necessary(p, p->mark());
379 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
380 }
381 _compact_point += obj_size;
382 }
383 };
384
385 class ShenandoahPrepareForCompactionTask : public WorkerTask {
386 private:
387 PreservedMarksSet* const _preserved_marks;
388 ShenandoahHeap* const _heap;
389 ShenandoahHeapRegionSet** const _worker_slices;
390
391 public:
392 ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
393 WorkerTask("Shenandoah Prepare For Compaction"),
394 _preserved_marks(preserved_marks),
395 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
396 }
397
398 static bool is_candidate_region(ShenandoahHeapRegion* r) {
399 // Empty region: get it into the slice to defragment the slice itself.
400 // We could have skipped this without violating correctness, but we really
401 // want to compact all live regions to the start of the heap, which sometimes
402 // means moving them into the fully empty regions.
403 if (r->is_empty()) return true;
404
405 // Can move the region, and this is not the humongous region. Humongous
406 // moves are special cased here, because their moves are handled separately.
407 return r->is_stw_move_allowed() && !r->is_humongous();
408 }
409
410 void work(uint worker_id) override;
411 private:
412 template<typename ClosureType>
413 void prepare_for_compaction(ClosureType& cl,
414 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
415 ShenandoahHeapRegionSetIterator& it,
416 ShenandoahHeapRegion* from_region);
417 };
418
419 void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
420 ShenandoahParallelWorkerSession worker_session(worker_id);
421 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
422 ShenandoahHeapRegionSetIterator it(slice);
423 ShenandoahHeapRegion* from_region = it.next();
424 // No work?
425 if (from_region == nullptr) {
426 return;
427 }
428
429 // Sliding compaction. Walk all regions in the slice, and compact them.
430 // Remember empty regions and reuse them as needed.
431 ResourceMark rm;
432
433 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
434
435 if (_heap->mode()->is_generational()) {
436 ShenandoahPrepareForGenerationalCompactionObjectClosure cl(_preserved_marks->get(worker_id),
437 empty_regions, from_region, worker_id);
438 prepare_for_compaction(cl, empty_regions, it, from_region);
439 } else {
440 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
441 prepare_for_compaction(cl, empty_regions, it, from_region);
442 }
443 }
444
445 template<typename ClosureType>
446 void ShenandoahPrepareForCompactionTask::prepare_for_compaction(ClosureType& cl,
447 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
448 ShenandoahHeapRegionSetIterator& it,
449 ShenandoahHeapRegion* from_region) {
450 while (from_region != nullptr) {
451 assert(is_candidate_region(from_region), "Sanity");
452 cl.set_from_region(from_region);
453 if (from_region->has_live()) {
454 _heap->marked_object_iterate(from_region, &cl);
455 }
456
457 // Compacted the region to somewhere else? From-region is empty then.
458 if (!cl.is_compact_same_region()) {
459 empty_regions.append(from_region);
460 }
461 from_region = it.next();
462 }
463 cl.finish();
464
465 // Mark all remaining regions as empty
466 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
467 ShenandoahHeapRegion* r = empty_regions.at(pos);
468 r->set_new_top(r->bottom());
469 }
470 }
471
472 void ShenandoahFullGC::calculate_target_humongous_objects() {
473 ShenandoahHeap* heap = ShenandoahHeap::heap();
474
475 // Compute the new addresses for humongous objects. We need to do this after addresses
476 // for regular objects are calculated, and we know what regions in heap suffix are
477 // available for humongous moves.
478 //
479 // Scan the heap backwards, because we are compacting humongous regions towards the end.
480 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
481 // humongous start there.
482 //
483 // The complication is potential non-movable regions during the scan. If such region is
484 // detected, then sliding restarts towards that non-movable region.
485
486 size_t to_begin = heap->num_regions();
487 size_t to_end = heap->num_regions();
488
489 log_debug(gc)("Full GC calculating target humongous objects from end %zu", to_end);
490 for (size_t c = heap->num_regions(); c > 0; c--) {
491 ShenandoahHeapRegion *r = heap->get_region(c - 1);
492 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
493 // To-region candidate: record this, and continue scan
494 to_begin = r->index();
495 continue;
496 }
497
498 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
499 // From-region candidate: movable humongous region
500 oop old_obj = cast_to_oop(r->bottom());
501 size_t words_size = old_obj->size();
502 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
503
504 size_t start = to_end - num_regions;
505
506 if (start >= to_begin && start != r->index()) {
507 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
508 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
509 FullGCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
510 to_end = start;
511 continue;
512 }
513 }
514
515 // Failed to fit. Scan starting from current region.
516 to_begin = r->index();
517 to_end = r->index();
518 }
519 }
520
521 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
522 public:
523 void heap_region_do(ShenandoahHeapRegion* r) override {
524 if (r->is_trash()) {
525 r->try_recycle_under_lock();
526 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
527 }
528 if (r->is_cset()) {
529 // Leave affiliation unchanged
530 r->make_regular_bypass();
531 }
532 if (r->is_empty_uncommitted()) {
533 r->make_committed_bypass();
534 }
535 assert (r->is_committed(), "only committed regions in heap now, see region %zu", r->index());
536
537 // Record current region occupancy: this communicates empty regions are free
538 // to the rest of Full GC code.
539 r->set_new_top(r->top());
540 }
541 };
542
543 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
544 private:
545 ShenandoahHeap* const _heap;
546 ShenandoahMarkingContext* const _ctx;
547
548 public:
549 ShenandoahTrashImmediateGarbageClosure() :
550 _heap(ShenandoahHeap::heap()),
551 _ctx(ShenandoahHeap::heap()->global_generation()->complete_marking_context()) {}
552
553 void heap_region_do(ShenandoahHeapRegion* r) override {
554 if (r->is_humongous_start()) {
555 oop humongous_obj = cast_to_oop(r->bottom());
556 if (!_ctx->is_marked(humongous_obj)) {
557 assert(!r->has_live(), "Region %zu is not marked, should not have live", r->index());
558 _heap->trash_humongous_region_at(r);
559 } else {
560 assert(r->has_live(), "Region %zu should have live", r->index());
561 }
562 } else if (r->is_humongous_continuation()) {
563 // If we hit continuation, the non-live humongous starts should have been trashed already
564 assert(r->humongous_start_region()->has_live(), "Region %zu should have live", r->index());
565 } else if (r->is_regular()) {
566 if (!r->has_live()) {
567 r->make_trash_immediate();
568 }
569 }
570 }
571 };
572
573 void ShenandoahFullGC::distribute_slices(ShenandoahHeapRegionSet** worker_slices) {
574 ShenandoahHeap* heap = ShenandoahHeap::heap();
575
576 uint n_workers = heap->workers()->active_workers();
577 size_t n_regions = heap->num_regions();
578
579 // What we want to accomplish: have the dense prefix of data, while still balancing
580 // out the parallel work.
581 //
582 // Assuming the amount of work is driven by the live data that needs moving, we can slice
583 // the entire heap into equal-live-sized prefix slices, and compact into them. So, each
584 // thread takes all regions in its prefix subset, and then it takes some regions from
585 // the tail.
586 //
587 // Tail region selection becomes interesting.
588 //
589 // First, we want to distribute the regions fairly between the workers, and those regions
590 // might have different amount of live data. So, until we sure no workers need live data,
591 // we need to only take what the worker needs.
592 //
593 // Second, since we slide everything to the left in each slice, the most busy regions
594 // would be the ones on the left. Which means we want to have all workers have their after-tail
595 // regions as close to the left as possible.
596 //
597 // The easiest way to do this is to distribute after-tail regions in round-robin between
598 // workers that still need live data.
599 //
600 // Consider parallel workers A, B, C, then the target slice layout would be:
601 //
602 // AAAAAAAABBBBBBBBCCCCCCCC|ABCABCABCABCABCABCABCABABABABABABABABABABAAAAA
603 //
604 // (.....dense-prefix.....) (.....................tail...................)
605 // [all regions fully live] [left-most regions are fuller that right-most]
606 //
607
608 // Compute how much live data is there. This would approximate the size of dense prefix
609 // we target to create.
610 size_t total_live = 0;
611 for (size_t idx = 0; idx < n_regions; idx++) {
612 ShenandoahHeapRegion *r = heap->get_region(idx);
613 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
614 total_live += r->get_live_data_words();
615 }
616 }
617
618 // Estimate the size for the dense prefix. Note that we specifically count only the
619 // "full" regions, so there would be some non-full regions in the slice tail.
620 size_t live_per_worker = total_live / n_workers;
621 size_t prefix_regions_per_worker = live_per_worker / ShenandoahHeapRegion::region_size_words();
622 size_t prefix_regions_total = prefix_regions_per_worker * n_workers;
623 prefix_regions_total = MIN2(prefix_regions_total, n_regions);
624 assert(prefix_regions_total <= n_regions, "Sanity");
625
626 // There might be non-candidate regions in the prefix. To compute where the tail actually
627 // ends up being, we need to account those as well.
628 size_t prefix_end = prefix_regions_total;
629 for (size_t idx = 0; idx < prefix_regions_total; idx++) {
630 ShenandoahHeapRegion *r = heap->get_region(idx);
631 if (!ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
632 prefix_end++;
633 }
634 }
635 prefix_end = MIN2(prefix_end, n_regions);
636 assert(prefix_end <= n_regions, "Sanity");
637
638 // Distribute prefix regions per worker: each thread definitely gets its own same-sized
639 // subset of dense prefix.
640 size_t prefix_idx = 0;
641
642 size_t* live = NEW_C_HEAP_ARRAY(size_t, n_workers, mtGC);
643
644 for (size_t wid = 0; wid < n_workers; wid++) {
645 ShenandoahHeapRegionSet* slice = worker_slices[wid];
646
647 live[wid] = 0;
648 size_t regs = 0;
649
650 // Add all prefix regions for this worker
651 while (prefix_idx < prefix_end && regs < prefix_regions_per_worker) {
652 ShenandoahHeapRegion *r = heap->get_region(prefix_idx);
653 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
654 slice->add_region(r);
655 live[wid] += r->get_live_data_words();
656 regs++;
657 }
658 prefix_idx++;
659 }
660 }
661
662 // Distribute the tail among workers in round-robin fashion.
663 size_t wid = n_workers - 1;
664
665 for (size_t tail_idx = prefix_end; tail_idx < n_regions; tail_idx++) {
666 ShenandoahHeapRegion *r = heap->get_region(tail_idx);
667 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
668 assert(wid < n_workers, "Sanity");
669
670 size_t live_region = r->get_live_data_words();
671
672 // Select next worker that still needs live data.
673 size_t old_wid = wid;
674 do {
675 wid++;
676 if (wid == n_workers) wid = 0;
677 } while (live[wid] + live_region >= live_per_worker && old_wid != wid);
678
679 if (old_wid == wid) {
680 // Circled back to the same worker? This means liveness data was
681 // miscalculated. Bump the live_per_worker limit so that
682 // everyone gets a piece of the leftover work.
683 live_per_worker += ShenandoahHeapRegion::region_size_words();
684 }
685
686 worker_slices[wid]->add_region(r);
687 live[wid] += live_region;
688 }
689 }
690
691 FREE_C_HEAP_ARRAY(size_t, live);
692
693 #ifdef ASSERT
694 ResourceBitMap map(n_regions);
695 for (size_t wid = 0; wid < n_workers; wid++) {
696 ShenandoahHeapRegionSetIterator it(worker_slices[wid]);
697 ShenandoahHeapRegion* r = it.next();
698 while (r != nullptr) {
699 size_t idx = r->index();
700 assert(ShenandoahPrepareForCompactionTask::is_candidate_region(r), "Sanity: %zu", idx);
701 assert(!map.at(idx), "No region distributed twice: %zu", idx);
702 map.at_put(idx, true);
703 r = it.next();
704 }
705 }
706
707 for (size_t rid = 0; rid < n_regions; rid++) {
708 bool is_candidate = ShenandoahPrepareForCompactionTask::is_candidate_region(heap->get_region(rid));
709 bool is_distributed = map.at(rid);
710 assert(is_distributed || !is_candidate, "All candidates are distributed: %zu", rid);
711 }
712 #endif
713 }
714
715 void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
716 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
717 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
718
719 ShenandoahHeap* heap = ShenandoahHeap::heap();
720
721 // About to figure out which regions can be compacted, make sure pinning status
722 // had been updated in GC prologue.
723 heap->assert_pinned_region_status();
724
725 {
726 // Trash the immediately collectible regions before computing addresses
727 ShenandoahTrashImmediateGarbageClosure trash_immediate_garbage;
728 ShenandoahExcludeRegionClosure<FREE> cl(&trash_immediate_garbage);
729 heap->heap_region_iterate(&cl);
730
731 // Make sure regions are in good state: committed, active, clean.
732 // This is needed because we are potentially sliding the data through them.
733 ShenandoahEnsureHeapActiveClosure ecl;
734 heap->heap_region_iterate(&ecl);
735 }
736
737 // Compute the new addresses for regular objects
738 {
739 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
740
741 distribute_slices(worker_slices);
742
743 ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices);
744 heap->workers()->run_task(&task);
745 }
746
747 // Compute the new addresses for humongous objects
748 {
749 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
750 calculate_target_humongous_objects();
751 }
752 }
753
754 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
755 private:
756 ShenandoahMarkingContext* const _ctx;
757
758 template <class T>
759 inline void do_oop_work(T* p) {
760 T o = RawAccess<>::oop_load(p);
761 if (!CompressedOops::is_null(o)) {
762 oop obj = CompressedOops::decode_not_null(o);
763 assert(_ctx->is_marked(obj), "must be marked");
764 if (FullGCForwarding::is_forwarded(obj)) {
765 oop forw = FullGCForwarding::forwardee(obj);
766 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
767 }
768 }
769 }
770
771 public:
772 ShenandoahAdjustPointersClosure() :
773 _ctx(ShenandoahHeap::heap()->global_generation()->complete_marking_context()) {}
774
775 void do_oop(oop* p) { do_oop_work(p); }
776 void do_oop(narrowOop* p) { do_oop_work(p); }
777 void do_method(Method* m) {}
778 void do_nmethod(nmethod* nm) {}
779 };
780
781 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
782 private:
783 ShenandoahAdjustPointersClosure _cl;
784
785 public:
786 void do_object(oop p) override {
787 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be complete");
788 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
789 p->oop_iterate(&_cl);
790 }
791 };
792
793 class ShenandoahAdjustPointersTask : public WorkerTask {
794 private:
795 ShenandoahHeap* const _heap;
796 ShenandoahRegionIterator _regions;
797
798 public:
799 ShenandoahAdjustPointersTask() :
800 WorkerTask("Shenandoah Adjust Pointers"),
801 _heap(ShenandoahHeap::heap()) {
802 }
803
804 void work(uint worker_id) override {
805 ShenandoahParallelWorkerSession worker_session(worker_id);
806 ShenandoahAdjustPointersObjectClosure obj_cl;
807 ShenandoahHeapRegion* r = _regions.next();
808 while (r != nullptr) {
809 if (!r->is_humongous_continuation() && r->has_live()) {
810 _heap->marked_object_iterate(r, &obj_cl);
811 }
812 if (_heap->mode()->is_generational()) {
813 ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(r);
814 }
815 r = _regions.next();
816 }
817 }
818 };
819
820 class ShenandoahAdjustRootPointersTask : public WorkerTask {
821 private:
822 ShenandoahRootAdjuster* _rp;
823 PreservedMarksSet* _preserved_marks;
824 public:
825 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
826 WorkerTask("Shenandoah Adjust Root Pointers"),
827 _rp(rp),
828 _preserved_marks(preserved_marks) {}
829
830 void work(uint worker_id) override {
831 ShenandoahParallelWorkerSession worker_session(worker_id);
832 ShenandoahAdjustPointersClosure cl;
833 _rp->roots_do(worker_id, &cl);
834 _preserved_marks->get(worker_id)->adjust_during_full_gc();
835 }
836 };
837
838 void ShenandoahFullGC::phase3_update_references() {
839 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
840 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
841
842 ShenandoahHeap* heap = ShenandoahHeap::heap();
843
844 WorkerThreads* workers = heap->workers();
845 uint nworkers = workers->active_workers();
846 {
847 #if COMPILER2_OR_JVMCI
848 DerivedPointerTable::clear();
849 #endif
850 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_adjust_roots);
851 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
852 workers->run_task(&task);
853 #if COMPILER2_OR_JVMCI
854 DerivedPointerTable::update_pointers();
855 #endif
856 }
857
858 ShenandoahAdjustPointersTask adjust_pointers_task;
859 workers->run_task(&adjust_pointers_task);
860 }
861
862 class ShenandoahCompactObjectsClosure : public ObjectClosure {
863 private:
864 uint const _worker_id;
865
866 public:
867 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
868 _worker_id(worker_id) {}
869
870 void do_object(oop p) override {
871 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
872 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
873 size_t size = p->size();
874 if (FullGCForwarding::is_forwarded(p)) {
875 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
876 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
877 assert(compact_from != compact_to, "Forwarded object should move");
878 Copy::aligned_conjoint_words(compact_from, compact_to, size);
879 oop new_obj = cast_to_oop(compact_to);
880
881 ContinuationGCSupport::relativize_stack_chunk(new_obj);
882 new_obj->init_mark();
883 }
884 }
885 };
886
887 class ShenandoahCompactObjectsTask : public WorkerTask {
888 private:
889 ShenandoahHeap* const _heap;
890 ShenandoahHeapRegionSet** const _worker_slices;
891
892 public:
893 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
894 WorkerTask("Shenandoah Compact Objects"),
895 _heap(ShenandoahHeap::heap()),
896 _worker_slices(worker_slices) {
897 }
898
899 void work(uint worker_id) override {
900 ShenandoahParallelWorkerSession worker_session(worker_id);
901 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
902
903 ShenandoahCompactObjectsClosure cl(worker_id);
904 ShenandoahHeapRegion* r = slice.next();
905 while (r != nullptr) {
906 assert(!r->is_humongous(), "must not get humongous regions here");
907 if (r->has_live()) {
908 _heap->marked_object_iterate(r, &cl);
909 }
910 r->set_top(r->new_top());
911 r = slice.next();
912 }
913 }
914 };
915
916 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
917 private:
918 ShenandoahHeap* const _heap;
919 bool _is_generational;
920 size_t _young_regions, _young_usage, _young_humongous_waste;
921 size_t _old_regions, _old_usage, _old_humongous_waste;
922
923 public:
924 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()),
925 _is_generational(_heap->mode()->is_generational()),
926 _young_regions(0),
927 _young_usage(0),
928 _young_humongous_waste(0),
929 _old_regions(0),
930 _old_usage(0),
931 _old_humongous_waste(0)
932 {
933 _heap->free_set()->clear();
934 }
935
936 void heap_region_do(ShenandoahHeapRegion* r) override {
937 assert (!r->is_cset(), "cset regions should have been demoted already");
938
939 // Need to reset the complete-top-at-mark-start pointer here because
940 // the complete marking bitmap is no longer valid. This ensures
941 // size-based iteration in marked_object_iterate().
942 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
943 // pinned regions.
944 if (!r->is_pinned()) {
945 _heap->marking_context()->reset_top_at_mark_start(r);
946 }
947
948 size_t live = r->used();
949
950 // Make empty regions that have been allocated into regular
951 if (r->is_empty() && live > 0) {
952 if (!_is_generational) {
953 r->make_affiliated_maybe();
954 }
955 // else, generational mode compaction has already established affiliation.
956 r->make_regular_bypass();
957 if (ZapUnusedHeapArea) {
958 SpaceMangler::mangle_region(MemRegion(r->top(), r->end()));
959 }
960 }
961
962 // Reclaim regular regions that became empty
963 if (r->is_regular() && live == 0) {
964 r->make_trash();
965 }
966
967 // Recycle all trash regions
968 if (r->is_trash()) {
969 live = 0;
970 r->try_recycle_under_lock();
971 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
972 } else {
973 if (r->is_old()) {
974 ShenandoahGenerationalFullGC::account_for_region(r, _old_regions, _old_usage, _old_humongous_waste);
975 } else if (r->is_young()) {
976 ShenandoahGenerationalFullGC::account_for_region(r, _young_regions, _young_usage, _young_humongous_waste);
977 }
978 }
979 r->set_live_data(live);
980 r->reset_alloc_metadata();
981 }
982 };
983
984 void ShenandoahFullGC::compact_humongous_objects() {
985 // Compact humongous regions, based on their fwdptr objects.
986 //
987 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
988 // humongous regions are already compacted, and do not require further moves, which alleviates
989 // sliding costs. We may consider doing this in parallel in the future.
990
991 ShenandoahHeap* heap = ShenandoahHeap::heap();
992
993 for (size_t c = heap->num_regions(); c > 0; c--) {
994 ShenandoahHeapRegion* r = heap->get_region(c - 1);
995 if (r->is_humongous_start()) {
996 oop old_obj = cast_to_oop(r->bottom());
997 if (!FullGCForwarding::is_forwarded(old_obj)) {
998 // No need to move the object, it stays at the same slot
999 continue;
1000 }
1001 size_t words_size = old_obj->size();
1002 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1003
1004 size_t old_start = r->index();
1005 size_t old_end = old_start + num_regions - 1;
1006 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1007 size_t new_end = new_start + num_regions - 1;
1008 assert(old_start != new_start, "must be real move");
1009 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1010
1011 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1012 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
1013 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1014
1015 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1016 new_obj->init_mark();
1017
1018 {
1019 ShenandoahAffiliation original_affiliation = r->affiliation();
1020 for (size_t c = old_start; c <= old_end; c++) {
1021 ShenandoahHeapRegion* r = heap->get_region(c);
1022 // Leave humongous region affiliation unchanged.
1023 r->make_regular_bypass();
1024 r->set_top(r->bottom());
1025 }
1026
1027 for (size_t c = new_start; c <= new_end; c++) {
1028 ShenandoahHeapRegion* r = heap->get_region(c);
1029 if (c == new_start) {
1030 r->make_humongous_start_bypass(original_affiliation);
1031 } else {
1032 r->make_humongous_cont_bypass(original_affiliation);
1033 }
1034
1035 // Trailing region may be non-full, record the remainder there
1036 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
1037 if ((c == new_end) && (remainder != 0)) {
1038 r->set_top(r->bottom() + remainder);
1039 } else {
1040 r->set_top(r->end());
1041 }
1042
1043 r->reset_alloc_metadata();
1044 }
1045 }
1046 }
1047 }
1048 }
1049
1050 // This is slightly different to ShHeap::reset_next_mark_bitmap:
1051 // we need to remain able to walk pinned regions.
1052 // Since pinned region do not move and don't get compacted, we will get holes with
1053 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
1054 // cannot be iterated over using oop->size()). The only way to safely iterate over those is using
1055 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
1056 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
1057 class ShenandoahMCResetCompleteBitmapTask : public WorkerTask {
1058 private:
1059 ShenandoahRegionIterator _regions;
1060
1061 public:
1062 ShenandoahMCResetCompleteBitmapTask() :
1063 WorkerTask("Shenandoah Reset Bitmap") {
1064 }
1065
1066 void work(uint worker_id) override {
1067 ShenandoahParallelWorkerSession worker_session(worker_id);
1068 ShenandoahHeapRegion* region = _regions.next();
1069 ShenandoahHeap* heap = ShenandoahHeap::heap();
1070 ShenandoahMarkingContext* const ctx = heap->marking_context();
1071 assert(heap->global_generation()->is_mark_complete(), "Marking must be complete");
1072 while (region != nullptr) {
1073 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
1074 ctx->clear_bitmap(region);
1075 }
1076 region = _regions.next();
1077 }
1078 }
1079 };
1080
1081 void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
1082 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
1083 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
1084
1085 ShenandoahHeap* heap = ShenandoahHeap::heap();
1086
1087 // Compact regular objects first
1088 {
1089 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
1090 ShenandoahCompactObjectsTask compact_task(worker_slices);
1091 heap->workers()->run_task(&compact_task);
1092 }
1093
1094 // Compact humongous objects after regular object moves
1095 {
1096 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
1097 compact_humongous_objects();
1098 }
1099 }
1100
1101 void ShenandoahFullGC::phase5_epilog() {
1102 GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer);
1103 ShenandoahHeap* heap = ShenandoahHeap::heap();
1104
1105 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
1106 // and must ensure the bitmap is in sync.
1107 {
1108 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
1109 ShenandoahMCResetCompleteBitmapTask task;
1110 heap->workers()->run_task(&task);
1111 }
1112
1113 // Bring regions in proper states after the collection, and set heap properties.
1114 {
1115 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
1116 ShenandoahPostCompactClosure post_compact;
1117 heap->heap_region_iterate(&post_compact);
1118 heap->collection_set()->clear();
1119 {
1120 ShenandoahFreeSet* free_set = heap->free_set();
1121 size_t young_trashed_regions, old_trashed_regions, first_old, last_old, num_old;
1122 free_set->prepare_to_rebuild(young_trashed_regions, old_trashed_regions, first_old, last_old, num_old);
1123 // We also do not expand old generation size following Full GC because we have scrambled age populations and
1124 // no longer have objects separated by age into distinct regions.
1125 if (heap->mode()->is_generational()) {
1126 ShenandoahGenerationalFullGC::compute_balances();
1127 }
1128 heap->free_set()->finish_rebuild(young_trashed_regions, old_trashed_regions, num_old);
1129 }
1130
1131 // Set mark incomplete because the marking bitmaps have been reset except pinned regions.
1132 _generation->set_mark_incomplete();
1133
1134 heap->clear_cancelled_gc();
1135 }
1136
1137 _preserved_marks->restore(heap->workers());
1138 _preserved_marks->reclaim();
1139
1140 if (heap->mode()->is_generational()) {
1141 ShenandoahGenerationalFullGC::rebuild_remembered_set(heap);
1142 }
1143 }