1 /*
2 * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/continuationGCSupport.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/preservedMarks.inline.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "gc/shared/workerThread.hpp"
34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
35 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
36 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
38 #include "gc/shenandoah/shenandoahFullGC.hpp"
39 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
41 #include "gc/shenandoah/shenandoahMark.inline.hpp"
42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
43 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
46 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
47 #include "gc/shenandoah/shenandoahMetrics.hpp"
48 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
49 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
50 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
51 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
52 #include "gc/shenandoah/shenandoahSTWMark.hpp"
53 #include "gc/shenandoah/shenandoahUtils.hpp"
54 #include "gc/shenandoah/shenandoahVerifier.hpp"
55 #include "gc/shenandoah/shenandoahVMOperations.hpp"
56 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
57 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
58 #include "memory/metaspaceUtils.hpp"
59 #include "memory/universe.hpp"
60 #include "oops/compressedOops.inline.hpp"
61 #include "oops/oop.inline.hpp"
62 #include "runtime/javaThread.hpp"
63 #include "runtime/orderAccess.hpp"
64 #include "runtime/vmThread.hpp"
65 #include "utilities/copy.hpp"
66 #include "utilities/events.hpp"
67 #include "utilities/growableArray.hpp"
68
69 // After Full GC is done, reconstruct the remembered set by iterating over OLD regions,
70 // registering all objects between bottom() and top(), and setting remembered set cards to
71 // DIRTY if they hold interesting pointers.
72 class ShenandoahReconstructRememberedSetTask : public WorkerTask {
73 private:
74 ShenandoahRegionIterator _regions;
75
76 public:
77 ShenandoahReconstructRememberedSetTask() :
78 WorkerTask("Shenandoah Reset Bitmap") { }
79
80 void work(uint worker_id) {
81 ShenandoahParallelWorkerSession worker_session(worker_id);
82 ShenandoahHeapRegion* r = _regions.next();
83 ShenandoahHeap* heap = ShenandoahHeap::heap();
84 RememberedScanner* scanner = heap->card_scan();
85 ShenandoahSetRememberedCardsToDirtyClosure dirty_cards_for_interesting_pointers;
86
87 while (r != nullptr) {
88 if (r->is_old() && r->is_active()) {
89 HeapWord* obj_addr = r->bottom();
90 if (r->is_humongous_start()) {
91 // First, clear the remembered set
92 oop obj = cast_to_oop(obj_addr);
93 size_t size = obj->size();
94
95 // First, clear the remembered set for all spanned humongous regions
96 size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
97 size_t region_span = num_regions * ShenandoahHeapRegion::region_size_words();
98 scanner->reset_remset(r->bottom(), region_span);
99 size_t region_index = r->index();
100 ShenandoahHeapRegion* humongous_region = heap->get_region(region_index);
101 while (num_regions-- != 0) {
102 scanner->reset_object_range(humongous_region->bottom(), humongous_region->end());
103 region_index++;
104 humongous_region = heap->get_region(region_index);
105 }
106
107 // Then register the humongous object and DIRTY relevant remembered set cards
108 scanner->register_object_without_lock(obj_addr);
109 obj->oop_iterate(&dirty_cards_for_interesting_pointers);
110 } else if (!r->is_humongous()) {
111 // First, clear the remembered set
112 scanner->reset_remset(r->bottom(), ShenandoahHeapRegion::region_size_words());
113 scanner->reset_object_range(r->bottom(), r->end());
114
115 // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards
116 HeapWord* t = r->top();
117 while (obj_addr < t) {
118 oop obj = cast_to_oop(obj_addr);
119 size_t size = obj->size();
120 scanner->register_object_without_lock(obj_addr);
121 obj_addr += obj->oop_iterate_size(&dirty_cards_for_interesting_pointers);
122 }
123 } // else, ignore humongous continuation region
124 }
125 // else, this region is FREE or YOUNG or inactive and we can ignore it.
126 // TODO: Assert this.
127 r = _regions.next();
128 }
129 }
130 };
131
132 ShenandoahFullGC::ShenandoahFullGC() :
133 _gc_timer(ShenandoahHeap::heap()->gc_timer()),
134 _preserved_marks(new PreservedMarksSet(true)) {}
135
136 ShenandoahFullGC::~ShenandoahFullGC() {
137 delete _preserved_marks;
138 }
139
140 bool ShenandoahFullGC::collect(GCCause::Cause cause) {
141 vmop_entry_full(cause);
142 // Always success
143 return true;
144 }
145
146 void ShenandoahFullGC::vmop_entry_full(GCCause::Cause cause) {
147 ShenandoahHeap* const heap = ShenandoahHeap::heap();
148 TraceCollectorStats tcs(heap->monitoring_support()->full_stw_collection_counters());
149 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
150
151 heap->try_inject_alloc_failure();
152 VM_ShenandoahFullGC op(cause, this);
153 VMThread::execute(&op);
154 }
155
156 void ShenandoahFullGC::entry_full(GCCause::Cause cause) {
157 static const char* msg = "Pause Full";
158 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
159 EventMark em("%s", msg);
160
161 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
162 ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
163 "full gc");
164
165 op_full(cause);
166 }
167
168 void ShenandoahFullGC::op_full(GCCause::Cause cause) {
169 ShenandoahHeap* const heap = ShenandoahHeap::heap();
170 ShenandoahMetricsSnapshot metrics;
171 metrics.snap_before();
172
173 // Perform full GC
174 do_it(cause);
175
176 metrics.snap_after();
177 if (heap->mode()->is_generational()) {
178 heap->mmu_tracker()->record_full(heap->global_generation(), GCId::current());
179 heap->log_heap_status("At end of Full GC");
180
181 // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are
182 // made valid by the time Full GC completes.
183 assert(heap->old_generation()->used_regions_size() <= heap->old_generation()->max_capacity(),
184 "Old generation affiliated regions must be less than capacity");
185 assert(heap->young_generation()->used_regions_size() <= heap->young_generation()->max_capacity(),
186 "Young generation affiliated regions must be less than capacity");
187
188 assert((heap->young_generation()->used() + heap->young_generation()->get_humongous_waste())
189 <= heap->young_generation()->used_regions_size(), "Young consumed can be no larger than span of affiliated regions");
190 assert((heap->old_generation()->used() + heap->old_generation()->get_humongous_waste())
191 <= heap->old_generation()->used_regions_size(), "Old consumed can be no larger than span of affiliated regions");
192
193 }
194 if (metrics.is_good_progress()) {
195 ShenandoahHeap::heap()->notify_gc_progress();
196 } else {
197 // Nothing to do. Tell the allocation path that we have failed to make
198 // progress, and it can finally fail.
199 ShenandoahHeap::heap()->notify_gc_no_progress();
200 }
201 }
202
203 void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
204 ShenandoahHeap* heap = ShenandoahHeap::heap();
205 // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL.
206 heap->set_gc_generation(heap->global_generation());
207
208 if (heap->mode()->is_generational()) {
209 // No need for old_gen->increase_used() as this was done when plabs were allocated.
210 heap->set_young_evac_reserve(0);
211 heap->set_old_evac_reserve(0);
212 heap->reset_old_evac_expended();
213 heap->set_promoted_reserve(0);
214
215 // Full GC supersedes any marking or coalescing in old generation.
216 heap->cancel_old_gc();
217 }
218
219 if (ShenandoahVerify) {
220 heap->verifier()->verify_before_fullgc();
221 }
222
223 if (VerifyBeforeGC) {
224 Universe::verify();
225 }
226
227 // Degenerated GC may carry concurrent root flags when upgrading to
228 // full GC. We need to reset it before mutators resume.
229 heap->set_concurrent_strong_root_in_progress(false);
230 heap->set_concurrent_weak_root_in_progress(false);
231
232 heap->set_full_gc_in_progress(true);
233
234 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
235 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
236
237 {
238 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_pre);
239 heap->pre_full_gc_dump(_gc_timer);
240 }
241
242 {
243 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
244 // Full GC is supposed to recover from any GC state:
245
246 // a0. Remember if we have forwarded objects
247 bool has_forwarded_objects = heap->has_forwarded_objects();
248
249 // a1. Cancel evacuation, if in progress
250 if (heap->is_evacuation_in_progress()) {
251 heap->set_evacuation_in_progress(false);
252 }
253 assert(!heap->is_evacuation_in_progress(), "sanity");
254
255 // a2. Cancel update-refs, if in progress
256 if (heap->is_update_refs_in_progress()) {
257 heap->set_update_refs_in_progress(false);
258 }
259 assert(!heap->is_update_refs_in_progress(), "sanity");
260
261 // b. Cancel all concurrent marks, if in progress
262 if (heap->is_concurrent_mark_in_progress()) {
263 heap->cancel_concurrent_mark();
264 }
265 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
266
267 // c. Update roots if this full GC is due to evac-oom, which may carry from-space pointers in roots.
268 if (has_forwarded_objects) {
269 update_roots(true /*full_gc*/);
270 }
271
272 // d. Reset the bitmaps for new marking
273 heap->global_generation()->reset_mark_bitmap();
274 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
275 assert(!heap->global_generation()->is_mark_complete(), "sanity");
276
277 // e. Abandon reference discovery and clear all discovered references.
278 ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor();
279 rp->abandon_partial_discovery();
280
281 // f. Sync pinned region status from the CP marks
282 heap->sync_pinned_region_status();
283
284 if (heap->mode()->is_generational()) {
285 for (size_t i = 0; i < heap->num_regions(); i++) {
286 ShenandoahHeapRegion* r = heap->get_region(i);
287 if (r->get_top_before_promote() != nullptr) {
288 r->restore_top_before_promote();
289 }
290 }
291 }
292
293 // The rest of prologue:
294 _preserved_marks->init(heap->workers()->active_workers());
295
296 assert(heap->has_forwarded_objects() == has_forwarded_objects, "This should not change");
297 }
298
299 if (UseTLAB) {
300 // TODO: Do we need to explicitly retire PLABs?
301 heap->gclabs_retire(ResizeTLAB);
302 heap->tlabs_retire(ResizeTLAB);
303 }
304
305 OrderAccess::fence();
306
307 phase1_mark_heap();
308
309 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
310 // Coming out of Full GC, we would not have any forwarded objects.
311 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
312 heap->set_has_forwarded_objects(false);
313
314 heap->set_full_gc_move_in_progress(true);
315
316 // Setup workers for the rest
317 OrderAccess::fence();
318
319 // Initialize worker slices
320 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
321 for (uint i = 0; i < heap->max_workers(); i++) {
322 worker_slices[i] = new ShenandoahHeapRegionSet();
323 }
324
325 {
326 // The rest of code performs region moves, where region status is undefined
327 // until all phases run together.
328 ShenandoahHeapLocker lock(heap->lock());
329
330 phase2_calculate_target_addresses(worker_slices);
331
332 OrderAccess::fence();
333
334 phase3_update_references();
335
336 phase4_compact_objects(worker_slices);
337
338 phase5_epilog();
339 }
340
341 {
342 // Epilogue
343 // TODO: Merge with phase5_epilog?
344 _preserved_marks->restore(heap->workers());
345 _preserved_marks->reclaim();
346
347 if (heap->mode()->is_generational()) {
348 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_reconstruct_remembered_set);
349 ShenandoahReconstructRememberedSetTask task;
350 heap->workers()->run_task(&task);
351 }
352 }
353
354 // Resize metaspace
355 MetaspaceGC::compute_new_size();
356
357 // Free worker slices
358 for (uint i = 0; i < heap->max_workers(); i++) {
359 delete worker_slices[i];
360 }
361 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
362
363 heap->set_full_gc_move_in_progress(false);
364 heap->set_full_gc_in_progress(false);
365
366 if (ShenandoahVerify) {
367 heap->verifier()->verify_after_fullgc();
368 }
369
370 // Humongous regions are promoted on demand and are accounted for by normal Full GC mechanisms.
371 if (VerifyAfterGC) {
372 Universe::verify();
373 }
374
375 {
376 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdump_post);
377 heap->post_full_gc_dump(_gc_timer);
378 }
379 }
380
381 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
382 private:
383 ShenandoahMarkingContext* const _ctx;
384
385 public:
386 ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
387
388 void heap_region_do(ShenandoahHeapRegion *r) {
389 if (r->affiliation() != FREE) {
390 _ctx->capture_top_at_mark_start(r);
391 r->clear_live_data();
392 }
393 }
394
395 bool is_thread_safe() { return true; }
396 };
397
398 void ShenandoahFullGC::phase1_mark_heap() {
399 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
400 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
401
402 ShenandoahHeap* heap = ShenandoahHeap::heap();
403
404 ShenandoahPrepareForMarkClosure cl;
405 heap->parallel_heap_region_iterate(&cl);
406
407 heap->set_unload_classes(heap->global_generation()->heuristics()->can_unload_classes());
408
409 ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor();
410 // enable ("weak") refs discovery
411 rp->set_soft_reference_policy(true); // forcefully purge all soft references
412
413 ShenandoahSTWMark mark(heap->global_generation(), true /*full_gc*/);
414 mark.mark();
415 heap->parallel_cleaning(true /* full_gc */);
416
417 size_t live_bytes_in_old = 0;
418 for (size_t i = 0; i < heap->num_regions(); i++) {
419 ShenandoahHeapRegion* r = heap->get_region(i);
420 if (r->is_old()) {
421 live_bytes_in_old += r->get_live_data_bytes();
422 }
423 }
424 log_info(gc)("Live bytes in old after STW mark: " PROPERFMT, PROPERFMTARGS(live_bytes_in_old));
425 heap->old_generation()->set_live_bytes_after_last_mark(live_bytes_in_old);
426 }
427
428 class ShenandoahPrepareForCompactionTask : public WorkerTask {
429 private:
430 PreservedMarksSet* const _preserved_marks;
431 ShenandoahHeap* const _heap;
432 ShenandoahHeapRegionSet** const _worker_slices;
433 size_t const _num_workers;
434
435 public:
436 ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks,
437 ShenandoahHeapRegionSet **worker_slices,
438 size_t num_workers);
439
440 static bool is_candidate_region(ShenandoahHeapRegion* r) {
441 // Empty region: get it into the slice to defragment the slice itself.
442 // We could have skipped this without violating correctness, but we really
443 // want to compact all live regions to the start of the heap, which sometimes
444 // means moving them into the fully empty regions.
445 if (r->is_empty()) return true;
446
447 // Can move the region, and this is not the humongous region. Humongous
448 // moves are special cased here, because their moves are handled separately.
449 return r->is_stw_move_allowed() && !r->is_humongous();
450 }
451
452 void work(uint worker_id);
453 };
454
455 class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClosure {
456 private:
457 PreservedMarks* const _preserved_marks;
458 ShenandoahHeap* const _heap;
459 uint _tenuring_threshold;
460
461 // _empty_regions is a thread-local list of heap regions that have been completely emptied by this worker thread's
462 // compaction efforts. The worker thread that drives these efforts adds compacted regions to this list if the
463 // region has not been compacted onto itself.
464 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
465 int _empty_regions_pos;
466 ShenandoahHeapRegion* _old_to_region;
467 ShenandoahHeapRegion* _young_to_region;
468 ShenandoahHeapRegion* _from_region;
469 ShenandoahAffiliation _from_affiliation;
470 HeapWord* _old_compact_point;
471 HeapWord* _young_compact_point;
472 uint _worker_id;
473
474 public:
475 ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks,
476 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
477 ShenandoahHeapRegion* old_to_region,
478 ShenandoahHeapRegion* young_to_region, uint worker_id) :
479 _preserved_marks(preserved_marks),
480 _heap(ShenandoahHeap::heap()),
481 _tenuring_threshold(0),
482 _empty_regions(empty_regions),
483 _empty_regions_pos(0),
484 _old_to_region(old_to_region),
485 _young_to_region(young_to_region),
486 _from_region(nullptr),
487 _old_compact_point((old_to_region != nullptr)? old_to_region->bottom(): nullptr),
488 _young_compact_point((young_to_region != nullptr)? young_to_region->bottom(): nullptr),
489 _worker_id(worker_id) {
490 if (_heap->mode()->is_generational()) {
491 _tenuring_threshold = _heap->age_census()->tenuring_threshold();
492 }
493 }
494
495 void set_from_region(ShenandoahHeapRegion* from_region) {
496 _from_region = from_region;
497 _from_affiliation = from_region->affiliation();
498 if (_from_region->has_live()) {
499 if (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION) {
500 if (_old_to_region == nullptr) {
501 _old_to_region = from_region;
502 _old_compact_point = from_region->bottom();
503 }
504 } else {
505 assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG");
506 if (_young_to_region == nullptr) {
507 _young_to_region = from_region;
508 _young_compact_point = from_region->bottom();
509 }
510 }
511 } // else, we won't iterate over this _from_region so we don't need to set up to region to hold copies
512 }
513
514 void finish() {
515 finish_old_region();
516 finish_young_region();
517 }
518
519 void finish_old_region() {
520 if (_old_to_region != nullptr) {
521 log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u",
522 _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id);
523 _old_to_region->set_new_top(_old_compact_point);
524 _old_to_region = nullptr;
525 }
526 }
527
528 void finish_young_region() {
529 if (_young_to_region != nullptr) {
530 log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT,
531 _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom());
532 _young_to_region->set_new_top(_young_compact_point);
533 _young_to_region = nullptr;
534 }
535 }
536
537 bool is_compact_same_region() {
538 return (_from_region == _old_to_region) || (_from_region == _young_to_region);
539 }
540
541 int empty_regions_pos() {
542 return _empty_regions_pos;
543 }
544
545 void do_object(oop p) {
546 assert(_from_region != nullptr, "must set before work");
547 assert((_from_region->bottom() <= cast_from_oop<HeapWord*>(p)) && (cast_from_oop<HeapWord*>(p) < _from_region->top()),
548 "Object must reside in _from_region");
549 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
550 assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
551
552 size_t obj_size = p->size();
553 uint from_region_age = _from_region->age();
554 uint object_age = p->age();
555
556 bool promote_object = false;
557 if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) &&
558 (from_region_age + object_age >= _tenuring_threshold)) {
559 if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) {
560 finish_old_region();
561 _old_to_region = nullptr;
562 }
563 if (_old_to_region == nullptr) {
564 if (_empty_regions_pos < _empty_regions.length()) {
565 ShenandoahHeapRegion* new_to_region = _empty_regions.at(_empty_regions_pos);
566 _empty_regions_pos++;
567 new_to_region->set_affiliation(OLD_GENERATION);
568 _old_to_region = new_to_region;
569 _old_compact_point = _old_to_region->bottom();
570 promote_object = true;
571 }
572 // Else this worker thread does not yet have any empty regions into which this aged object can be promoted so
573 // we leave promote_object as false, deferring the promotion.
574 } else {
575 promote_object = true;
576 }
577 }
578
579 if (promote_object || (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION)) {
580 assert(_old_to_region != nullptr, "_old_to_region should not be nullptr when evacuating to OLD region");
581 if (_old_compact_point + obj_size > _old_to_region->end()) {
582 ShenandoahHeapRegion* new_to_region;
583
584 log_debug(gc)("Worker %u finishing old region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
585 ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _old_to_region->index(),
586 p2i(_old_compact_point), obj_size, p2i(_old_compact_point + obj_size), p2i(_old_to_region->end()));
587
588 // Object does not fit. Get a new _old_to_region.
589 finish_old_region();
590 if (_empty_regions_pos < _empty_regions.length()) {
591 new_to_region = _empty_regions.at(_empty_regions_pos);
592 _empty_regions_pos++;
593 new_to_region->set_affiliation(OLD_GENERATION);
594 } else {
595 // If we've exhausted the previously selected _old_to_region, we know that the _old_to_region is distinct
596 // from _from_region. That's because there is always room for _from_region to be compacted into itself.
597 // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
598 new_to_region = _from_region;
599 }
600
601 assert(new_to_region != _old_to_region, "must not reuse same OLD to-region");
602 assert(new_to_region != nullptr, "must not be nullptr");
603 _old_to_region = new_to_region;
604 _old_compact_point = _old_to_region->bottom();
605 }
606
607 // Object fits into current region, record new location:
608 assert(_old_compact_point + obj_size <= _old_to_region->end(), "must fit");
609 shenandoah_assert_not_forwarded(nullptr, p);
610 _preserved_marks->push_if_necessary(p, p->mark());
611 p->forward_to(cast_to_oop(_old_compact_point));
612 _old_compact_point += obj_size;
613 } else {
614 assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION,
615 "_from_region must be OLD_GENERATION or YOUNG_GENERATION");
616 assert(_young_to_region != nullptr, "_young_to_region should not be nullptr when compacting YOUNG _from_region");
617
618 // After full gc compaction, all regions have age 0. Embed the region's age into the object's age in order to preserve
619 // tenuring progress.
620 if (_heap->is_aging_cycle()) {
621 _heap->increase_object_age(p, from_region_age + 1);
622 } else {
623 _heap->increase_object_age(p, from_region_age);
624 }
625
626 if (_young_compact_point + obj_size > _young_to_region->end()) {
627 ShenandoahHeapRegion* new_to_region;
628
629 log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
630 ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _young_to_region->index(),
631 p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end()));
632
633 // Object does not fit. Get a new _young_to_region.
634 finish_young_region();
635 if (_empty_regions_pos < _empty_regions.length()) {
636 new_to_region = _empty_regions.at(_empty_regions_pos);
637 _empty_regions_pos++;
638 new_to_region->set_affiliation(YOUNG_GENERATION);
639 } else {
640 // If we've exhausted the previously selected _young_to_region, we know that the _young_to_region is distinct
641 // from _from_region. That's because there is always room for _from_region to be compacted into itself.
642 // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
643 new_to_region = _from_region;
644 }
645
646 assert(new_to_region != _young_to_region, "must not reuse same OLD to-region");
647 assert(new_to_region != nullptr, "must not be nullptr");
648 _young_to_region = new_to_region;
649 _young_compact_point = _young_to_region->bottom();
650 }
651
652 // Object fits into current region, record new location:
653 assert(_young_compact_point + obj_size <= _young_to_region->end(), "must fit");
654 shenandoah_assert_not_forwarded(nullptr, p);
655 _preserved_marks->push_if_necessary(p, p->mark());
656 p->forward_to(cast_to_oop(_young_compact_point));
657 _young_compact_point += obj_size;
658 }
659 }
660 };
661
662
663 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
664 private:
665 PreservedMarks* const _preserved_marks;
666 ShenandoahHeap* const _heap;
667 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
668 int _empty_regions_pos;
669 ShenandoahHeapRegion* _to_region;
670 ShenandoahHeapRegion* _from_region;
671 HeapWord* _compact_point;
672
673 public:
674 ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
675 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
676 ShenandoahHeapRegion* to_region) :
677 _preserved_marks(preserved_marks),
678 _heap(ShenandoahHeap::heap()),
679 _empty_regions(empty_regions),
680 _empty_regions_pos(0),
681 _to_region(to_region),
682 _from_region(nullptr),
683 _compact_point(to_region->bottom()) {}
684
685 void set_from_region(ShenandoahHeapRegion* from_region) {
686 _from_region = from_region;
687 }
688
689 void finish_region() {
690 assert(_to_region != nullptr, "should not happen");
691 assert(!_heap->mode()->is_generational(), "Generational GC should use different Closure");
692 _to_region->set_new_top(_compact_point);
693 }
694
695 bool is_compact_same_region() {
696 return _from_region == _to_region;
697 }
698
699 int empty_regions_pos() {
700 return _empty_regions_pos;
701 }
702
703 void do_object(oop p) {
704 assert(_from_region != nullptr, "must set before work");
705 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
706 assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
707
708 size_t obj_size = p->size();
709 if (_compact_point + obj_size > _to_region->end()) {
710 finish_region();
711
712 // Object doesn't fit. Pick next empty region and start compacting there.
713 ShenandoahHeapRegion* new_to_region;
714 if (_empty_regions_pos < _empty_regions.length()) {
715 new_to_region = _empty_regions.at(_empty_regions_pos);
716 _empty_regions_pos++;
717 } else {
718 // Out of empty region? Compact within the same region.
719 new_to_region = _from_region;
720 }
721
722 assert(new_to_region != _to_region, "must not reuse same to-region");
723 assert(new_to_region != nullptr, "must not be null");
724 _to_region = new_to_region;
725 _compact_point = _to_region->bottom();
726 }
727
728 // Object fits into current region, record new location:
729 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
730 shenandoah_assert_not_forwarded(nullptr, p);
731 _preserved_marks->push_if_necessary(p, p->mark());
732 p->forward_to(cast_to_oop(_compact_point));
733 _compact_point += obj_size;
734 }
735 };
736
737
738 ShenandoahPrepareForCompactionTask::ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks,
739 ShenandoahHeapRegionSet **worker_slices,
740 size_t num_workers) :
741 WorkerTask("Shenandoah Prepare For Compaction"),
742 _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()),
743 _worker_slices(worker_slices), _num_workers(num_workers) { }
744
745
746 void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
747 ShenandoahParallelWorkerSession worker_session(worker_id);
748 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
749 ShenandoahHeapRegionSetIterator it(slice);
750 ShenandoahHeapRegion* from_region = it.next();
751 // No work?
752 if (from_region == nullptr) {
753 return;
754 }
755
756 // Sliding compaction. Walk all regions in the slice, and compact them.
757 // Remember empty regions and reuse them as needed.
758 ResourceMark rm;
759
760 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
761
762 if (_heap->mode()->is_generational()) {
763 ShenandoahHeapRegion* old_to_region = (from_region->is_old())? from_region: nullptr;
764 ShenandoahHeapRegion* young_to_region = (from_region->is_young())? from_region: nullptr;
765 ShenandoahPrepareForGenerationalCompactionObjectClosure cl(_preserved_marks->get(worker_id),
766 empty_regions,
767 old_to_region, young_to_region,
768 worker_id);
769 while (from_region != nullptr) {
770 assert(is_candidate_region(from_region), "Sanity");
771 log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live",
772 worker_id, from_region->affiliation_name(),
773 from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have");
774 cl.set_from_region(from_region);
775 if (from_region->has_live()) {
776 _heap->marked_object_iterate(from_region, &cl);
777 }
778 // Compacted the region to somewhere else? From-region is empty then.
779 if (!cl.is_compact_same_region()) {
780 empty_regions.append(from_region);
781 }
782 from_region = it.next();
783 }
784 cl.finish();
785
786 // Mark all remaining regions as empty
787 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
788 ShenandoahHeapRegion* r = empty_regions.at(pos);
789 r->set_new_top(r->bottom());
790 }
791 } else {
792 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
793 while (from_region != nullptr) {
794 assert(is_candidate_region(from_region), "Sanity");
795 cl.set_from_region(from_region);
796 if (from_region->has_live()) {
797 _heap->marked_object_iterate(from_region, &cl);
798 }
799
800 // Compacted the region to somewhere else? From-region is empty then.
801 if (!cl.is_compact_same_region()) {
802 empty_regions.append(from_region);
803 }
804 from_region = it.next();
805 }
806 cl.finish_region();
807
808 // Mark all remaining regions as empty
809 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
810 ShenandoahHeapRegion* r = empty_regions.at(pos);
811 r->set_new_top(r->bottom());
812 }
813 }
814 }
815
816 void ShenandoahFullGC::calculate_target_humongous_objects() {
817 ShenandoahHeap* heap = ShenandoahHeap::heap();
818
819 // Compute the new addresses for humongous objects. We need to do this after addresses
820 // for regular objects are calculated, and we know what regions in heap suffix are
821 // available for humongous moves.
822 //
823 // Scan the heap backwards, because we are compacting humongous regions towards the end.
824 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
825 // humongous start there.
826 //
827 // The complication is potential non-movable regions during the scan. If such region is
828 // detected, then sliding restarts towards that non-movable region.
829
830 size_t to_begin = heap->num_regions();
831 size_t to_end = heap->num_regions();
832
833 log_debug(gc)("Full GC calculating target humongous objects from end " SIZE_FORMAT, to_end);
834 for (size_t c = heap->num_regions(); c > 0; c--) {
835 ShenandoahHeapRegion *r = heap->get_region(c - 1);
836 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
837 // To-region candidate: record this, and continue scan
838 to_begin = r->index();
839 continue;
840 }
841
842 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
843 // From-region candidate: movable humongous region
844 oop old_obj = cast_to_oop(r->bottom());
845 size_t words_size = old_obj->size();
846 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
847
848 size_t start = to_end - num_regions;
849
850 if (start >= to_begin && start != r->index()) {
851 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
852 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
853 old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom()));
854 to_end = start;
855 continue;
856 }
857 }
858
859 // Failed to fit. Scan starting from current region.
860 to_begin = r->index();
861 to_end = r->index();
862 }
863 }
864
865 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
866 private:
867 ShenandoahHeap* const _heap;
868
869 public:
870 ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
871 void heap_region_do(ShenandoahHeapRegion* r) {
872 if (r->is_trash()) {
873 r->recycle();
874 }
875 if (r->is_cset()) {
876 // Leave affiliation unchanged
877 r->make_regular_bypass();
878 }
879 if (r->is_empty_uncommitted()) {
880 r->make_committed_bypass();
881 }
882 assert (r->is_committed(), "only committed regions in heap now, see region " SIZE_FORMAT, r->index());
883
884 // Record current region occupancy: this communicates empty regions are free
885 // to the rest of Full GC code.
886 r->set_new_top(r->top());
887 }
888 };
889
890 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
891 private:
892 ShenandoahHeap* const _heap;
893 ShenandoahMarkingContext* const _ctx;
894
895 public:
896 ShenandoahTrashImmediateGarbageClosure() :
897 _heap(ShenandoahHeap::heap()),
898 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
899
900 void heap_region_do(ShenandoahHeapRegion* r) {
901 if (!r->is_affiliated()) {
902 // Ignore free regions
903 // TODO: change iterators so they do not process FREE regions.
904 return;
905 }
906
907 if (r->is_humongous_start()) {
908 oop humongous_obj = cast_to_oop(r->bottom());
909 if (!_ctx->is_marked(humongous_obj)) {
910 assert(!r->has_live(),
911 "Humongous Start %s Region " SIZE_FORMAT " is not marked, should not have live",
912 r->affiliation_name(), r->index());
913 log_debug(gc)("Trashing immediate humongous region " SIZE_FORMAT " because not marked", r->index());
914 _heap->trash_humongous_region_at(r);
915 } else {
916 assert(r->has_live(),
917 "Humongous Start %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index());
918 }
919 } else if (r->is_humongous_continuation()) {
920 // If we hit continuation, the non-live humongous starts should have been trashed already
921 assert(r->humongous_start_region()->has_live(),
922 "Humongous Continuation %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index());
923 } else if (r->is_regular()) {
924 if (!r->has_live()) {
925 log_debug(gc)("Trashing immediate regular region " SIZE_FORMAT " because has no live", r->index());
926 r->make_trash_immediate();
927 }
928 }
929 }
930 };
931
932 void ShenandoahFullGC::distribute_slices(ShenandoahHeapRegionSet** worker_slices) {
933 ShenandoahHeap* heap = ShenandoahHeap::heap();
934
935 uint n_workers = heap->workers()->active_workers();
936 size_t n_regions = heap->num_regions();
937
938 // What we want to accomplish: have the dense prefix of data, while still balancing
939 // out the parallel work.
940 //
941 // Assuming the amount of work is driven by the live data that needs moving, we can slice
942 // the entire heap into equal-live-sized prefix slices, and compact into them. So, each
943 // thread takes all regions in its prefix subset, and then it takes some regions from
944 // the tail.
945 //
946 // Tail region selection becomes interesting.
947 //
948 // First, we want to distribute the regions fairly between the workers, and those regions
949 // might have different amount of live data. So, until we sure no workers need live data,
950 // we need to only take what the worker needs.
951 //
952 // Second, since we slide everything to the left in each slice, the most busy regions
953 // would be the ones on the left. Which means we want to have all workers have their after-tail
954 // regions as close to the left as possible.
955 //
956 // The easiest way to do this is to distribute after-tail regions in round-robin between
957 // workers that still need live data.
958 //
959 // Consider parallel workers A, B, C, then the target slice layout would be:
960 //
961 // AAAAAAAABBBBBBBBCCCCCCCC|ABCABCABCABCABCABCABCABABABABABABABABABABAAAAA
962 //
963 // (.....dense-prefix.....) (.....................tail...................)
964 // [all regions fully live] [left-most regions are fuller that right-most]
965 //
966
967 // Compute how much live data is there. This would approximate the size of dense prefix
968 // we target to create.
969 size_t total_live = 0;
970 for (size_t idx = 0; idx < n_regions; idx++) {
971 ShenandoahHeapRegion *r = heap->get_region(idx);
972 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
973 total_live += r->get_live_data_words();
974 }
975 }
976
977 // Estimate the size for the dense prefix. Note that we specifically count only the
978 // "full" regions, so there would be some non-full regions in the slice tail.
979 size_t live_per_worker = total_live / n_workers;
980 size_t prefix_regions_per_worker = live_per_worker / ShenandoahHeapRegion::region_size_words();
981 size_t prefix_regions_total = prefix_regions_per_worker * n_workers;
982 prefix_regions_total = MIN2(prefix_regions_total, n_regions);
983 assert(prefix_regions_total <= n_regions, "Sanity");
984
985 // There might be non-candidate regions in the prefix. To compute where the tail actually
986 // ends up being, we need to account those as well.
987 size_t prefix_end = prefix_regions_total;
988 for (size_t idx = 0; idx < prefix_regions_total; idx++) {
989 ShenandoahHeapRegion *r = heap->get_region(idx);
990 if (!ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
991 prefix_end++;
992 }
993 }
994 prefix_end = MIN2(prefix_end, n_regions);
995 assert(prefix_end <= n_regions, "Sanity");
996
997 // Distribute prefix regions per worker: each thread definitely gets its own same-sized
998 // subset of dense prefix.
999 size_t prefix_idx = 0;
1000
1001 size_t* live = NEW_C_HEAP_ARRAY(size_t, n_workers, mtGC);
1002
1003 for (size_t wid = 0; wid < n_workers; wid++) {
1004 ShenandoahHeapRegionSet* slice = worker_slices[wid];
1005
1006 live[wid] = 0;
1007 size_t regs = 0;
1008
1009 // Add all prefix regions for this worker
1010 while (prefix_idx < prefix_end && regs < prefix_regions_per_worker) {
1011 ShenandoahHeapRegion *r = heap->get_region(prefix_idx);
1012 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
1013 slice->add_region(r);
1014 live[wid] += r->get_live_data_words();
1015 regs++;
1016 }
1017 prefix_idx++;
1018 }
1019 }
1020
1021 // Distribute the tail among workers in round-robin fashion.
1022 size_t wid = n_workers - 1;
1023
1024 for (size_t tail_idx = prefix_end; tail_idx < n_regions; tail_idx++) {
1025 ShenandoahHeapRegion *r = heap->get_region(tail_idx);
1026 if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
1027 assert(wid < n_workers, "Sanity");
1028
1029 size_t live_region = r->get_live_data_words();
1030
1031 // Select next worker that still needs live data.
1032 size_t old_wid = wid;
1033 do {
1034 wid++;
1035 if (wid == n_workers) wid = 0;
1036 } while (live[wid] + live_region >= live_per_worker && old_wid != wid);
1037
1038 if (old_wid == wid) {
1039 // Circled back to the same worker? This means liveness data was
1040 // miscalculated. Bump the live_per_worker limit so that
1041 // everyone gets a piece of the leftover work.
1042 live_per_worker += ShenandoahHeapRegion::region_size_words();
1043 }
1044
1045 worker_slices[wid]->add_region(r);
1046 live[wid] += live_region;
1047 }
1048 }
1049
1050 FREE_C_HEAP_ARRAY(size_t, live);
1051
1052 #ifdef ASSERT
1053 ResourceBitMap map(n_regions);
1054 for (size_t wid = 0; wid < n_workers; wid++) {
1055 ShenandoahHeapRegionSetIterator it(worker_slices[wid]);
1056 ShenandoahHeapRegion* r = it.next();
1057 while (r != nullptr) {
1058 size_t idx = r->index();
1059 assert(ShenandoahPrepareForCompactionTask::is_candidate_region(r), "Sanity: " SIZE_FORMAT, idx);
1060 assert(!map.at(idx), "No region distributed twice: " SIZE_FORMAT, idx);
1061 map.at_put(idx, true);
1062 r = it.next();
1063 }
1064 }
1065
1066 for (size_t rid = 0; rid < n_regions; rid++) {
1067 bool is_candidate = ShenandoahPrepareForCompactionTask::is_candidate_region(heap->get_region(rid));
1068 bool is_distributed = map.at(rid);
1069 assert(is_distributed || !is_candidate, "All candidates are distributed: " SIZE_FORMAT, rid);
1070 }
1071 #endif
1072 }
1073
1074 // TODO:
1075 // Consider compacting old-gen objects toward the high end of memory and young-gen objects towards the low-end
1076 // of memory. As currently implemented, all regions are compacted toward the low-end of memory. This creates more
1077 // fragmentation of the heap, because old-gen regions get scattered among low-address regions such that it becomes
1078 // more difficult to find contiguous regions for humongous objects.
1079 void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
1080 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
1081 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
1082
1083 ShenandoahHeap* heap = ShenandoahHeap::heap();
1084
1085 // About to figure out which regions can be compacted, make sure pinning status
1086 // had been updated in GC prologue.
1087 heap->assert_pinned_region_status();
1088
1089 {
1090 // Trash the immediately collectible regions before computing addresses
1091 ShenandoahTrashImmediateGarbageClosure tigcl;
1092 heap->heap_region_iterate(&tigcl);
1093
1094 // Make sure regions are in good state: committed, active, clean.
1095 // This is needed because we are potentially sliding the data through them.
1096 ShenandoahEnsureHeapActiveClosure ecl;
1097 heap->heap_region_iterate(&ecl);
1098 }
1099
1100 // Compute the new addresses for regular objects
1101 {
1102 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
1103
1104 distribute_slices(worker_slices);
1105
1106 size_t num_workers = heap->max_workers();
1107
1108 ResourceMark rm;
1109 ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices, num_workers);
1110 heap->workers()->run_task(&task);
1111 }
1112
1113 // Compute the new addresses for humongous objects
1114 {
1115 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
1116 calculate_target_humongous_objects();
1117 }
1118 }
1119
1120 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
1121 private:
1122 ShenandoahHeap* const _heap;
1123 ShenandoahMarkingContext* const _ctx;
1124
1125 template <class T>
1126 inline void do_oop_work(T* p) {
1127 T o = RawAccess<>::oop_load(p);
1128 if (!CompressedOops::is_null(o)) {
1129 oop obj = CompressedOops::decode_not_null(o);
1130 assert(_ctx->is_marked(obj), "must be marked");
1131 if (obj->is_forwarded()) {
1132 oop forw = obj->forwardee();
1133 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
1134 }
1135 }
1136 }
1137
1138 public:
1139 ShenandoahAdjustPointersClosure() :
1140 _heap(ShenandoahHeap::heap()),
1141 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
1142
1143 void do_oop(oop* p) { do_oop_work(p); }
1144 void do_oop(narrowOop* p) { do_oop_work(p); }
1145 void do_method(Method* m) {}
1146 void do_nmethod(nmethod* nm) {}
1147 };
1148
1149 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
1150 private:
1151 ShenandoahHeap* const _heap;
1152 ShenandoahAdjustPointersClosure _cl;
1153
1154 public:
1155 ShenandoahAdjustPointersObjectClosure() :
1156 _heap(ShenandoahHeap::heap()) {
1157 }
1158 void do_object(oop p) {
1159 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
1160 p->oop_iterate(&_cl);
1161 }
1162 };
1163
1164 class ShenandoahAdjustPointersTask : public WorkerTask {
1165 private:
1166 ShenandoahHeap* const _heap;
1167 ShenandoahRegionIterator _regions;
1168
1169 public:
1170 ShenandoahAdjustPointersTask() :
1171 WorkerTask("Shenandoah Adjust Pointers"),
1172 _heap(ShenandoahHeap::heap()) {
1173 }
1174
1175 void work(uint worker_id) {
1176 ShenandoahParallelWorkerSession worker_session(worker_id);
1177 ShenandoahAdjustPointersObjectClosure obj_cl;
1178 ShenandoahHeapRegion* r = _regions.next();
1179 while (r != nullptr) {
1180 if (!r->is_humongous_continuation() && r->has_live()) {
1181 _heap->marked_object_iterate(r, &obj_cl);
1182 }
1183 if (r->is_pinned() && r->is_old() && r->is_active() && !r->is_humongous()) {
1184 // Pinned regions are not compacted so they may still hold unmarked objects with
1185 // reference to reclaimed memory. Remembered set scanning will crash if it attempts
1186 // to iterate the oops in these objects.
1187 r->begin_preemptible_coalesce_and_fill();
1188 r->oop_fill_and_coalesce_without_cancel();
1189 }
1190 r = _regions.next();
1191 }
1192 }
1193 };
1194
1195 class ShenandoahAdjustRootPointersTask : public WorkerTask {
1196 private:
1197 ShenandoahRootAdjuster* _rp;
1198 PreservedMarksSet* _preserved_marks;
1199 public:
1200 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
1201 WorkerTask("Shenandoah Adjust Root Pointers"),
1202 _rp(rp),
1203 _preserved_marks(preserved_marks) {}
1204
1205 void work(uint worker_id) {
1206 ShenandoahParallelWorkerSession worker_session(worker_id);
1207 ShenandoahAdjustPointersClosure cl;
1208 _rp->roots_do(worker_id, &cl);
1209 _preserved_marks->get(worker_id)->adjust_during_full_gc();
1210 }
1211 };
1212
1213 void ShenandoahFullGC::phase3_update_references() {
1214 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
1215 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
1216
1217 ShenandoahHeap* heap = ShenandoahHeap::heap();
1218
1219 WorkerThreads* workers = heap->workers();
1220 uint nworkers = workers->active_workers();
1221 {
1222 #if COMPILER2_OR_JVMCI
1223 DerivedPointerTable::clear();
1224 #endif
1225 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_adjust_roots);
1226 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
1227 workers->run_task(&task);
1228 #if COMPILER2_OR_JVMCI
1229 DerivedPointerTable::update_pointers();
1230 #endif
1231 }
1232
1233 ShenandoahAdjustPointersTask adjust_pointers_task;
1234 workers->run_task(&adjust_pointers_task);
1235 }
1236
1237 class ShenandoahCompactObjectsClosure : public ObjectClosure {
1238 private:
1239 ShenandoahHeap* const _heap;
1240 uint const _worker_id;
1241
1242 public:
1243 ShenandoahCompactObjectsClosure(uint worker_id) :
1244 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
1245
1246 void do_object(oop p) {
1247 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
1248 size_t size = p->size();
1249 if (p->is_forwarded()) {
1250 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
1251 HeapWord* compact_to = cast_from_oop<HeapWord*>(p->forwardee());
1252 Copy::aligned_conjoint_words(compact_from, compact_to, size);
1253 oop new_obj = cast_to_oop(compact_to);
1254
1255 ContinuationGCSupport::relativize_stack_chunk(new_obj);
1256 new_obj->init_mark();
1257 }
1258 }
1259 };
1260
1261 class ShenandoahCompactObjectsTask : public WorkerTask {
1262 private:
1263 ShenandoahHeap* const _heap;
1264 ShenandoahHeapRegionSet** const _worker_slices;
1265
1266 public:
1267 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
1268 WorkerTask("Shenandoah Compact Objects"),
1269 _heap(ShenandoahHeap::heap()),
1270 _worker_slices(worker_slices) {
1271 }
1272
1273 void work(uint worker_id) {
1274 ShenandoahParallelWorkerSession worker_session(worker_id);
1275 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
1276
1277 ShenandoahCompactObjectsClosure cl(worker_id);
1278 ShenandoahHeapRegion* r = slice.next();
1279 while (r != nullptr) {
1280 assert(!r->is_humongous(), "must not get humongous regions here");
1281 if (r->has_live()) {
1282 _heap->marked_object_iterate(r, &cl);
1283 }
1284 r->set_top(r->new_top());
1285 r = slice.next();
1286 }
1287 }
1288 };
1289
1290 static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) {
1291 region_count++;
1292 region_usage += r->used();
1293 if (r->is_humongous_start()) {
1294 // For each humongous object, we take this path once regardless of how many regions it spans.
1295 HeapWord* obj_addr = r->bottom();
1296 oop obj = cast_to_oop(obj_addr);
1297 size_t word_size = obj->size();
1298 size_t region_size_words = ShenandoahHeapRegion::region_size_words();
1299 size_t overreach = word_size % region_size_words;
1300 if (overreach != 0) {
1301 humongous_waste += (region_size_words - overreach) * HeapWordSize;
1302 }
1303 // else, this humongous object aligns exactly on region size, so no waste.
1304 }
1305 }
1306
1307 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
1308 private:
1309 ShenandoahHeap* const _heap;
1310 bool _is_generational;
1311 size_t _young_regions, _young_usage, _young_humongous_waste;
1312 size_t _old_regions, _old_usage, _old_humongous_waste;
1313
1314 public:
1315 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()),
1316 _is_generational(_heap->mode()->is_generational()),
1317 _young_regions(0),
1318 _young_usage(0),
1319 _young_humongous_waste(0),
1320 _old_regions(0),
1321 _old_usage(0),
1322 _old_humongous_waste(0)
1323 {
1324 _heap->free_set()->clear();
1325 }
1326
1327 void heap_region_do(ShenandoahHeapRegion* r) {
1328 assert (!r->is_cset(), "cset regions should have been demoted already");
1329
1330 // Need to reset the complete-top-at-mark-start pointer here because
1331 // the complete marking bitmap is no longer valid. This ensures
1332 // size-based iteration in marked_object_iterate().
1333 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
1334 // pinned regions.
1335 if (!r->is_pinned()) {
1336 _heap->complete_marking_context()->reset_top_at_mark_start(r);
1337 }
1338
1339 size_t live = r->used();
1340
1341 // Make empty regions that have been allocated into regular
1342 if (r->is_empty() && live > 0) {
1343 if (!_is_generational) {
1344 r->make_young_maybe();
1345 }
1346 // else, generational mode compaction has already established affiliation.
1347 r->make_regular_bypass();
1348 }
1349
1350 // Reclaim regular regions that became empty
1351 if (r->is_regular() && live == 0) {
1352 r->make_trash();
1353 }
1354
1355 // Recycle all trash regions
1356 if (r->is_trash()) {
1357 live = 0;
1358 r->recycle();
1359 } else {
1360 if (r->is_old()) {
1361 account_for_region(r, _old_regions, _old_usage, _old_humongous_waste);
1362 } else if (r->is_young()) {
1363 account_for_region(r, _young_regions, _young_usage, _young_humongous_waste);
1364 }
1365 }
1366 r->set_live_data(live);
1367 r->reset_alloc_metadata();
1368 }
1369
1370 void update_generation_usage() {
1371 if (_is_generational) {
1372 _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste);
1373 _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste);
1374 } else {
1375 assert(_old_regions == 0, "Old regions only expected in generational mode");
1376 assert(_old_usage == 0, "Old usage only expected in generational mode");
1377 assert(_old_humongous_waste == 0, "Old humongous waste only expected in generational mode");
1378 }
1379
1380 // In generational mode, global usage should be the sum of young and old. This is also true
1381 // for non-generational modes except that there are no old regions.
1382 _heap->global_generation()->establish_usage(_old_regions + _young_regions,
1383 _old_usage + _young_usage,
1384 _old_humongous_waste + _young_humongous_waste);
1385 }
1386 };
1387
1388 void ShenandoahFullGC::compact_humongous_objects() {
1389 // Compact humongous regions, based on their fwdptr objects.
1390 //
1391 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
1392 // humongous regions are already compacted, and do not require further moves, which alleviates
1393 // sliding costs. We may consider doing this in parallel in the future.
1394
1395 ShenandoahHeap* heap = ShenandoahHeap::heap();
1396
1397 for (size_t c = heap->num_regions(); c > 0; c--) {
1398 ShenandoahHeapRegion* r = heap->get_region(c - 1);
1399 if (r->is_humongous_start()) {
1400 oop old_obj = cast_to_oop(r->bottom());
1401 if (!old_obj->is_forwarded()) {
1402 // No need to move the object, it stays at the same slot
1403 continue;
1404 }
1405 size_t words_size = old_obj->size();
1406 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1407
1408 size_t old_start = r->index();
1409 size_t old_end = old_start + num_regions - 1;
1410 size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
1411 size_t new_end = new_start + num_regions - 1;
1412 assert(old_start != new_start, "must be real move");
1413 assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
1414
1415 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(heap->get_region(old_start)->bottom()));
1416 log_debug(gc)("Full GC compaction moves humongous object from region " SIZE_FORMAT " to region " SIZE_FORMAT,
1417 old_start, new_start);
1418
1419 Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
1420 heap->get_region(new_start)->bottom(),
1421 words_size);
1422
1423 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1424 new_obj->init_mark();
1425
1426 {
1427 ShenandoahAffiliation original_affiliation = r->affiliation();
1428 for (size_t c = old_start; c <= old_end; c++) {
1429 ShenandoahHeapRegion* r = heap->get_region(c);
1430 // Leave humongous region affiliation unchanged.
1431 r->make_regular_bypass();
1432 r->set_top(r->bottom());
1433 }
1434
1435 for (size_t c = new_start; c <= new_end; c++) {
1436 ShenandoahHeapRegion* r = heap->get_region(c);
1437 if (c == new_start) {
1438 r->make_humongous_start_bypass(original_affiliation);
1439 } else {
1440 r->make_humongous_cont_bypass(original_affiliation);
1441 }
1442
1443 // Trailing region may be non-full, record the remainder there
1444 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
1445 if ((c == new_end) && (remainder != 0)) {
1446 r->set_top(r->bottom() + remainder);
1447 } else {
1448 r->set_top(r->end());
1449 }
1450
1451 r->reset_alloc_metadata();
1452 }
1453 }
1454 }
1455 }
1456 }
1457
1458 // This is slightly different to ShHeap::reset_next_mark_bitmap:
1459 // we need to remain able to walk pinned regions.
1460 // Since pinned region do not move and don't get compacted, we will get holes with
1461 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
1462 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
1463 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
1464 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
1465 class ShenandoahMCResetCompleteBitmapTask : public WorkerTask {
1466 private:
1467 ShenandoahRegionIterator _regions;
1468
1469 public:
1470 ShenandoahMCResetCompleteBitmapTask() :
1471 WorkerTask("Shenandoah Reset Bitmap") {
1472 }
1473
1474 void work(uint worker_id) {
1475 ShenandoahParallelWorkerSession worker_session(worker_id);
1476 ShenandoahHeapRegion* region = _regions.next();
1477 ShenandoahHeap* heap = ShenandoahHeap::heap();
1478 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
1479 while (region != nullptr) {
1480 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
1481 ctx->clear_bitmap(region);
1482 }
1483 region = _regions.next();
1484 }
1485 }
1486 };
1487
1488 void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
1489 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
1490 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
1491
1492 ShenandoahHeap* heap = ShenandoahHeap::heap();
1493
1494 // Compact regular objects first
1495 {
1496 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
1497 ShenandoahCompactObjectsTask compact_task(worker_slices);
1498 heap->workers()->run_task(&compact_task);
1499 }
1500
1501 // Compact humongous objects after regular object moves
1502 {
1503 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
1504 compact_humongous_objects();
1505 }
1506 }
1507
1508 void ShenandoahFullGC::phase5_epilog() {
1509 GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer);
1510 ShenandoahHeap* heap = ShenandoahHeap::heap();
1511
1512 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
1513 // and must ensure the bitmap is in sync.
1514 {
1515 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
1516 ShenandoahMCResetCompleteBitmapTask task;
1517 heap->workers()->run_task(&task);
1518 }
1519
1520 // Bring regions in proper states after the collection, and set heap properties.
1521 {
1522 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
1523 ShenandoahPostCompactClosure post_compact;
1524 heap->heap_region_iterate(&post_compact);
1525 post_compact.update_generation_usage();
1526 if (heap->mode()->is_generational()) {
1527 size_t old_usage = heap->old_generation()->used_regions_size();
1528 size_t old_capacity = heap->old_generation()->max_capacity();
1529
1530 assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must aligh with region size");
1531 assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must aligh with region size");
1532
1533 if (old_capacity > old_usage) {
1534 size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes();
1535 heap->generation_sizer()->transfer_to_young(excess_old_regions);
1536 } else if (old_capacity < old_usage) {
1537 size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes();
1538 heap->generation_sizer()->force_transfer_to_old(old_regions_deficit);
1539 }
1540
1541 log_info(gc)("FullGC done: young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s",
1542 byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()),
1543 byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used()));
1544 }
1545 heap->collection_set()->clear();
1546 size_t young_cset_regions, old_cset_regions;
1547 heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions);
1548
1549 // We also do not expand old generation size following Full GC because we have scrambled age populations and
1550 // no longer have objects separated by age into distinct regions.
1551
1552 // TODO: Do we need to fix FullGC so that it maintains aged segregation of objects into distinct regions?
1553 // A partial solution would be to remember how many objects are of tenure age following Full GC, but
1554 // this is probably suboptimal, because most of these objects will not reside in a region that will be
1555 // selected for the next evacuation phase.
1556
1557 // In case this Full GC resulted from degeneration, clear the tally on anticipated promotion.
1558 heap->clear_promotion_potential();
1559
1560 if (heap->mode()->is_generational()) {
1561 // Invoke this in case we are able to transfer memory from OLD to YOUNG.
1562 heap->adjust_generation_sizes_for_next_cycle(0, 0, 0);
1563 }
1564 heap->free_set()->rebuild(young_cset_regions, old_cset_regions);
1565
1566 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
1567 // abbreviated cycle.
1568 if (heap->mode()->is_generational()) {
1569 bool success;
1570 size_t region_xfer;
1571 const char* region_destination;
1572 ShenandoahYoungGeneration* young_gen = heap->young_generation();
1573 ShenandoahGeneration* old_gen = heap->old_generation();
1574
1575 size_t old_region_surplus = heap->get_old_region_surplus();
1576 size_t old_region_deficit = heap->get_old_region_deficit();
1577 if (old_region_surplus) {
1578 success = heap->generation_sizer()->transfer_to_young(old_region_surplus);
1579 region_destination = "young";
1580 region_xfer = old_region_surplus;
1581 } else if (old_region_deficit) {
1582 success = heap->generation_sizer()->transfer_to_old(old_region_deficit);
1583 region_destination = "old";
1584 region_xfer = old_region_deficit;
1585 if (!success) {
1586 ((ShenandoahOldHeuristics *) old_gen->heuristics())->trigger_cannot_expand();
1587 }
1588 } else {
1589 region_destination = "none";
1590 region_xfer = 0;
1591 success = true;
1592 }
1593 heap->set_old_region_surplus(0);
1594 heap->set_old_region_deficit(0);
1595 size_t young_available = young_gen->available();
1596 size_t old_available = old_gen->available();
1597 log_info(gc, ergo)("After cleanup, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: "
1598 SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s",
1599 success? "successfully transferred": "failed to transfer", region_xfer, region_destination,
1600 byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available),
1601 byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available));
1602 }
1603 heap->clear_cancelled_gc(true /* clear oom handler */);
1604 }
1605 }