234 // Coming out of Full GC, we would not have any forwarded objects.
235 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
236 heap->set_has_forwarded_objects(false);
237
238 heap->set_full_gc_move_in_progress(true);
239
240 // Setup workers for the rest
241 OrderAccess::fence();
242
243 // Initialize worker slices
244 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
245 for (uint i = 0; i < heap->max_workers(); i++) {
246 worker_slices[i] = new ShenandoahHeapRegionSet();
247 }
248
249 {
250 // The rest of code performs region moves, where region status is undefined
251 // until all phases run together.
252 ShenandoahHeapLocker lock(heap->lock());
253
254 phase2_calculate_target_addresses(worker_slices);
255
256 OrderAccess::fence();
257
258 phase3_update_references();
259
260 phase4_compact_objects(worker_slices);
261
262 phase5_epilog();
263 }
264 heap->start_idle_span();
265
266 // Resize metaspace
267 MetaspaceGC::compute_new_size();
268
269 // Free worker slices
270 for (uint i = 0; i < heap->max_workers(); i++) {
271 delete worker_slices[i];
272 }
273 FREE_C_HEAP_ARRAY(worker_slices);
274
275 heap->set_full_gc_move_in_progress(false);
276 heap->set_full_gc_in_progress(false);
277
278 DEBUG_ONLY(heap->assert_no_self_forwards());
279
280 if (ShenandoahVerify) {
281 heap->verifier()->verify_after_fullgc(_generation);
282 }
345 void finish() {
346 assert(_to_region != nullptr, "should not happen");
347 _to_region->set_new_top(_compact_point);
348 }
349
350 bool is_compact_same_region() {
351 return _from_region == _to_region;
352 }
353
354 int empty_regions_pos() {
355 return _empty_regions_pos;
356 }
357
358 void do_object(oop p) override {
359 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
360 assert(_from_region != nullptr, "must set before work");
361 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
362 assert(_heap->marking_context()->is_marked(p), "must be marked");
363 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
364
365 size_t obj_size = p->size();
366 if (_compact_point + obj_size > _to_region->end()) {
367 finish();
368
369 // Object doesn't fit. Pick next empty region and start compacting there.
370 ShenandoahHeapRegion* new_to_region;
371 if (_empty_regions_pos < _empty_regions.length()) {
372 new_to_region = _empty_regions.at(_empty_regions_pos);
373 _empty_regions_pos++;
374 } else {
375 // Out of empty region? Compact within the same region.
376 new_to_region = _from_region;
377 }
378
379 assert(new_to_region != _to_region, "must not reuse same to-region");
380 assert(new_to_region != nullptr, "must not be null");
381 _to_region = new_to_region;
382 _compact_point = _to_region->bottom();
383 }
384
385 // Object fits into current region, record new location, if object does not move:
386 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
387 shenandoah_assert_not_forwarded(nullptr, p);
388 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
389 _preserved_marks->push_if_necessary(p, p->mark());
390 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
391 }
392 _compact_point += obj_size;
393 }
394 };
395
396 class ShenandoahPrepareForCompactionTask : public WorkerTask {
397 private:
398 PreservedMarksSet* const _preserved_marks;
399 ShenandoahHeap* const _heap;
400 ShenandoahHeapRegionSet** const _worker_slices;
401
402 public:
492 // humongous start there.
493 //
494 // The complication is potential non-movable regions during the scan. If such region is
495 // detected, then sliding restarts towards that non-movable region.
496
497 size_t to_begin = heap->num_regions();
498 size_t to_end = heap->num_regions();
499
500 log_debug(gc)("Full GC calculating target humongous objects from end %zu", to_end);
501 for (size_t c = heap->num_regions(); c > 0; c--) {
502 ShenandoahHeapRegion *r = heap->get_region(c - 1);
503 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
504 // To-region candidate: record this, and continue scan
505 to_begin = r->index();
506 continue;
507 }
508
509 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
510 // From-region candidate: movable humongous region
511 oop old_obj = cast_to_oop(r->bottom());
512 size_t words_size = old_obj->size();
513 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
514
515 size_t start = to_end - num_regions;
516
517 if (start >= to_begin && start != r->index()) {
518 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
519 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
520 FullGCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
521 to_end = start;
522 continue;
523 }
524 }
525
526 // Failed to fit. Scan starting from current region.
527 to_begin = r->index();
528 to_end = r->index();
529 }
530 }
531
532 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
533 public:
534 void heap_region_do(ShenandoahHeapRegion* r) override {
535 if (r->is_trash()) {
536 r->try_recycle_under_lock();
537 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
538 }
539 if (r->is_cset()) {
540 // Leave affiliation unchanged
541 r->make_regular_bypass();
542 }
875 uint const _worker_id;
876
877 public:
878 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
879 _worker_id(worker_id) {}
880
881 void do_object(oop p) override {
882 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
883 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
884 size_t size = p->size();
885 if (FullGCForwarding::is_forwarded(p)) {
886 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
887 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
888 assert(compact_from != compact_to, "Forwarded object should move");
889 Copy::aligned_conjoint_words(compact_from, compact_to, size);
890 oop new_obj = cast_to_oop(compact_to);
891
892 // Restore the mark word before relativizing the stack chunk. The copy's
893 // mark word contains the full GC forwarding encoding, which would cause
894 // is_stackChunk() to read garbage (especially with compact headers).
895 new_obj->init_mark();
896 ContinuationGCSupport::relativize_stack_chunk(new_obj);
897 }
898 }
899 };
900
901 class ShenandoahCompactObjectsTask : public WorkerTask {
902 private:
903 ShenandoahHeap* const _heap;
904 ShenandoahHeapRegionSet** const _worker_slices;
905
906 public:
907 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
908 WorkerTask("Shenandoah Compact Objects"),
909 _heap(ShenandoahHeap::heap()),
910 _worker_slices(worker_slices) {
911 }
912
913 void work(uint worker_id) override {
914 ShenandoahParallelWorkerSession worker_session(worker_id);
915 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
995 }
996 };
997
998 void ShenandoahFullGC::compact_humongous_objects() {
999 // Compact humongous regions, based on their fwdptr objects.
1000 //
1001 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
1002 // humongous regions are already compacted, and do not require further moves, which alleviates
1003 // sliding costs. We may consider doing this in parallel in the future.
1004
1005 ShenandoahHeap* heap = ShenandoahHeap::heap();
1006
1007 for (size_t c = heap->num_regions(); c > 0; c--) {
1008 ShenandoahHeapRegion* r = heap->get_region(c - 1);
1009 if (r->is_humongous_start()) {
1010 oop old_obj = cast_to_oop(r->bottom());
1011 if (!FullGCForwarding::is_forwarded(old_obj)) {
1012 // No need to move the object, it stays at the same slot
1013 continue;
1014 }
1015 size_t words_size = old_obj->size();
1016 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1017
1018 size_t old_start = r->index();
1019 size_t old_end = old_start + num_regions - 1;
1020 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1021 size_t new_end = new_start + num_regions - 1;
1022 assert(old_start != new_start, "must be real move");
1023 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1024
1025 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1026 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
1027 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1028
1029 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1030 new_obj->init_mark();
1031
1032 {
1033 ShenandoahAffiliation original_affiliation = r->affiliation();
1034 for (size_t c = old_start; c <= old_end; c++) {
1035 ShenandoahHeapRegion* r = heap->get_region(c);
1036 // Leave humongous region affiliation unchanged.
1037 r->make_regular_bypass();
1038 r->set_top(r->bottom());
1039 }
1040
1041 for (size_t c = new_start; c <= new_end; c++) {
1042 ShenandoahHeapRegion* r = heap->get_region(c);
1043 if (c == new_start) {
1044 r->make_humongous_start_bypass(original_affiliation);
1045 } else {
1046 r->make_humongous_cont_bypass(original_affiliation);
1047 }
1048
1049 // Trailing region may be non-full, record the remainder there
1050 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
1051 if ((c == new_end) && (remainder != 0)) {
1052 r->set_top(r->bottom() + remainder);
1053 } else {
1054 r->set_top(r->end());
1055 }
1056
1057 r->reset_alloc_metadata();
1058 }
1059 }
1060 }
1061 }
1062 }
1063
1064 // This is slightly different to ShHeap::reset_next_mark_bitmap:
1065 // we need to remain able to walk pinned regions.
1066 // Since pinned region do not move and don't get compacted, we will get holes with
1067 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
1068 // cannot be iterated over using oop->size()). The only way to safely iterate over those is using
1069 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
1070 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
|
234 // Coming out of Full GC, we would not have any forwarded objects.
235 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
236 heap->set_has_forwarded_objects(false);
237
238 heap->set_full_gc_move_in_progress(true);
239
240 // Setup workers for the rest
241 OrderAccess::fence();
242
243 // Initialize worker slices
244 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
245 for (uint i = 0; i < heap->max_workers(); i++) {
246 worker_slices[i] = new ShenandoahHeapRegionSet();
247 }
248
249 {
250 // The rest of code performs region moves, where region status is undefined
251 // until all phases run together.
252 ShenandoahHeapLocker lock(heap->lock());
253
254 FullGCForwarding::begin();
255
256 phase2_calculate_target_addresses(worker_slices);
257
258 OrderAccess::fence();
259
260 phase3_update_references();
261
262 phase4_compact_objects(worker_slices);
263
264 phase5_epilog();
265
266 FullGCForwarding::end();
267 }
268 heap->start_idle_span();
269
270 // Resize metaspace
271 MetaspaceGC::compute_new_size();
272
273 // Free worker slices
274 for (uint i = 0; i < heap->max_workers(); i++) {
275 delete worker_slices[i];
276 }
277 FREE_C_HEAP_ARRAY(worker_slices);
278
279 heap->set_full_gc_move_in_progress(false);
280 heap->set_full_gc_in_progress(false);
281
282 DEBUG_ONLY(heap->assert_no_self_forwards());
283
284 if (ShenandoahVerify) {
285 heap->verifier()->verify_after_fullgc(_generation);
286 }
349 void finish() {
350 assert(_to_region != nullptr, "should not happen");
351 _to_region->set_new_top(_compact_point);
352 }
353
354 bool is_compact_same_region() {
355 return _from_region == _to_region;
356 }
357
358 int empty_regions_pos() {
359 return _empty_regions_pos;
360 }
361
362 void do_object(oop p) override {
363 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
364 assert(_from_region != nullptr, "must set before work");
365 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
366 assert(_heap->marking_context()->is_marked(p), "must be marked");
367 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
368
369 size_t old_size = p->size();
370 size_t new_size = p->copy_size(old_size, p->mark());
371 size_t obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
372 if (_compact_point + obj_size > _to_region->end()) {
373 finish();
374
375 // Object doesn't fit. Pick next empty region and start compacting there.
376 ShenandoahHeapRegion* new_to_region;
377 if (_empty_regions_pos < _empty_regions.length()) {
378 new_to_region = _empty_regions.at(_empty_regions_pos);
379 _empty_regions_pos++;
380 } else {
381 // Out of empty region? Compact within the same region.
382 new_to_region = _from_region;
383 }
384
385 assert(new_to_region != _to_region, "must not reuse same to-region");
386 assert(new_to_region != nullptr, "must not be null");
387 _to_region = new_to_region;
388 _compact_point = _to_region->bottom();
389 obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
390 }
391
392 // Object fits into current region, record new location, if object does not move:
393 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
394 shenandoah_assert_not_forwarded(nullptr, p);
395 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
396 _preserved_marks->push_if_necessary(p, p->mark());
397 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
398 }
399 _compact_point += obj_size;
400 }
401 };
402
403 class ShenandoahPrepareForCompactionTask : public WorkerTask {
404 private:
405 PreservedMarksSet* const _preserved_marks;
406 ShenandoahHeap* const _heap;
407 ShenandoahHeapRegionSet** const _worker_slices;
408
409 public:
499 // humongous start there.
500 //
501 // The complication is potential non-movable regions during the scan. If such region is
502 // detected, then sliding restarts towards that non-movable region.
503
504 size_t to_begin = heap->num_regions();
505 size_t to_end = heap->num_regions();
506
507 log_debug(gc)("Full GC calculating target humongous objects from end %zu", to_end);
508 for (size_t c = heap->num_regions(); c > 0; c--) {
509 ShenandoahHeapRegion *r = heap->get_region(c - 1);
510 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
511 // To-region candidate: record this, and continue scan
512 to_begin = r->index();
513 continue;
514 }
515
516 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
517 // From-region candidate: movable humongous region
518 oop old_obj = cast_to_oop(r->bottom());
519 size_t new_words_size = old_obj->copy_size(old_obj->size(), old_obj->mark());
520 size_t num_regions = ShenandoahHeapRegion::required_regions(new_words_size * HeapWordSize);
521
522 // Test the fit before computing the slide target. With compact object headers
523 // the expanded size can require one region more than the object currently
524 // occupies, so num_regions may exceed the available window. Comparing against
525 // the window size (to_end - to_begin) avoids the unsigned underflow that
526 // "to_end - num_regions" would suffer when the object does not fit.
527 if (num_regions <= to_end - to_begin) {
528 size_t start = to_end - num_regions;
529 if (start != r->index()) {
530 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
531 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
532 FullGCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
533 to_end = start;
534 continue;
535 }
536 }
537 }
538
539 // Failed to fit. Scan starting from current region.
540 to_begin = r->index();
541 to_end = r->index();
542 }
543 }
544
545 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
546 public:
547 void heap_region_do(ShenandoahHeapRegion* r) override {
548 if (r->is_trash()) {
549 r->try_recycle_under_lock();
550 // No need to adjust_interval_for_recycled_old_region. That will be taken care of during freeset rebuild.
551 }
552 if (r->is_cset()) {
553 // Leave affiliation unchanged
554 r->make_regular_bypass();
555 }
888 uint const _worker_id;
889
890 public:
891 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
892 _worker_id(worker_id) {}
893
894 void do_object(oop p) override {
895 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
896 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
897 size_t size = p->size();
898 if (FullGCForwarding::is_forwarded(p)) {
899 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
900 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
901 assert(compact_from != compact_to, "Forwarded object should move");
902 Copy::aligned_conjoint_words(compact_from, compact_to, size);
903 oop new_obj = cast_to_oop(compact_to);
904
905 // Restore the mark word before relativizing the stack chunk. The copy's
906 // mark word contains the full GC forwarding encoding, which would cause
907 // is_stackChunk() to read garbage (especially with compact headers).
908 new_obj->reinit_mark();
909 new_obj->initialize_hash_if_necessary(p);
910 ContinuationGCSupport::relativize_stack_chunk(new_obj);
911 }
912 }
913 };
914
915 class ShenandoahCompactObjectsTask : public WorkerTask {
916 private:
917 ShenandoahHeap* const _heap;
918 ShenandoahHeapRegionSet** const _worker_slices;
919
920 public:
921 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
922 WorkerTask("Shenandoah Compact Objects"),
923 _heap(ShenandoahHeap::heap()),
924 _worker_slices(worker_slices) {
925 }
926
927 void work(uint worker_id) override {
928 ShenandoahParallelWorkerSession worker_session(worker_id);
929 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
1009 }
1010 };
1011
1012 void ShenandoahFullGC::compact_humongous_objects() {
1013 // Compact humongous regions, based on their fwdptr objects.
1014 //
1015 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
1016 // humongous regions are already compacted, and do not require further moves, which alleviates
1017 // sliding costs. We may consider doing this in parallel in the future.
1018
1019 ShenandoahHeap* heap = ShenandoahHeap::heap();
1020
1021 for (size_t c = heap->num_regions(); c > 0; c--) {
1022 ShenandoahHeapRegion* r = heap->get_region(c - 1);
1023 if (r->is_humongous_start()) {
1024 oop old_obj = cast_to_oop(r->bottom());
1025 if (!FullGCForwarding::is_forwarded(old_obj)) {
1026 // No need to move the object, it stays at the same slot
1027 continue;
1028 }
1029 size_t old_words_size = old_obj->size();
1030 size_t new_words_size = old_obj->copy_size(old_words_size, old_obj->mark());
1031 size_t old_num_regions = ShenandoahHeapRegion::required_regions(old_words_size * HeapWordSize);
1032 size_t new_num_regions = ShenandoahHeapRegion::required_regions(new_words_size * HeapWordSize);
1033
1034 size_t old_start = r->index();
1035 size_t old_end = old_start + old_num_regions - 1;
1036 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1037 size_t new_end = new_start + new_num_regions - 1;
1038 assert(old_start != new_start, "must be real move");
1039 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1040
1041 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1042 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), old_words_size);
1043 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1044
1045 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1046 new_obj->reinit_mark();
1047 new_obj->initialize_hash_if_necessary(old_obj);
1048
1049 {
1050 ShenandoahAffiliation original_affiliation = r->affiliation();
1051 for (size_t c = old_start; c <= old_end; c++) {
1052 ShenandoahHeapRegion* r = heap->get_region(c);
1053 // Leave humongous region affiliation unchanged.
1054 r->make_regular_bypass();
1055 r->set_top(r->bottom());
1056 }
1057
1058 for (size_t c = new_start; c <= new_end; c++) {
1059 ShenandoahHeapRegion* r = heap->get_region(c);
1060 if (c == new_start) {
1061 r->make_humongous_start_bypass(original_affiliation);
1062 } else {
1063 r->make_humongous_cont_bypass(original_affiliation);
1064 }
1065
1066 // Trailing region may be non-full, record the remainder there
1067 size_t remainder = new_words_size & ShenandoahHeapRegion::region_size_words_mask();
1068 if ((c == new_end) && (remainder != 0)) {
1069 r->set_top(r->bottom() + remainder);
1070 } else {
1071 r->set_top(r->end());
1072 }
1073
1074 r->reset_alloc_metadata();
1075 }
1076 }
1077 }
1078 }
1079 }
1080
1081 // This is slightly different to ShHeap::reset_next_mark_bitmap:
1082 // we need to remain able to walk pinned regions.
1083 // Since pinned region do not move and don't get compacted, we will get holes with
1084 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
1085 // cannot be iterated over using oop->size()). The only way to safely iterate over those is using
1086 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
1087 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
|