234 // Coming out of Full GC, we would not have any forwarded objects.
235 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
236 heap->set_has_forwarded_objects(false);
237
238 heap->set_full_gc_move_in_progress(true);
239
240 // Setup workers for the rest
241 OrderAccess::fence();
242
243 // Initialize worker slices
244 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
245 for (uint i = 0; i < heap->max_workers(); i++) {
246 worker_slices[i] = new ShenandoahHeapRegionSet();
247 }
248
249 {
250 // The rest of code performs region moves, where region status is undefined
251 // until all phases run together.
252 ShenandoahHeapLocker lock(heap->lock());
253
254 phase2_calculate_target_addresses(worker_slices);
255
256 OrderAccess::fence();
257
258 phase3_update_references();
259
260 phase4_compact_objects(worker_slices);
261
262 phase5_epilog();
263 }
264 heap->start_idle_span();
265
266 // Resize metaspace
267 MetaspaceGC::compute_new_size();
268
269 // Free worker slices
270 for (uint i = 0; i < heap->max_workers(); i++) {
271 delete worker_slices[i];
272 }
273 FREE_C_HEAP_ARRAY(worker_slices);
274
275 heap->set_full_gc_move_in_progress(false);
276 heap->set_full_gc_in_progress(false);
277
278 DEBUG_ONLY(heap->assert_no_self_forwards());
279
280 if (ShenandoahVerify) {
281 heap->verifier()->verify_after_fullgc(_generation);
282 }
345 void finish() {
346 assert(_to_region != nullptr, "should not happen");
347 _to_region->set_new_top(_compact_point);
348 }
349
350 bool is_compact_same_region() {
351 return _from_region == _to_region;
352 }
353
354 int empty_regions_pos() {
355 return _empty_regions_pos;
356 }
357
358 void do_object(oop p) override {
359 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
360 assert(_from_region != nullptr, "must set before work");
361 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
362 assert(_heap->marking_context()->is_marked(p), "must be marked");
363 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
364
365 size_t obj_size = p->size();
366 if (_compact_point + obj_size > _to_region->end()) {
367 finish();
368
369 // Object doesn't fit. Pick next empty region and start compacting there.
370 ShenandoahHeapRegion* new_to_region;
371 if (_empty_regions_pos < _empty_regions.length()) {
372 new_to_region = _empty_regions.at(_empty_regions_pos);
373 _empty_regions_pos++;
374 } else {
375 // Out of empty region? Compact within the same region.
376 new_to_region = _from_region;
377 }
378
379 assert(new_to_region != _to_region, "must not reuse same to-region");
380 assert(new_to_region != nullptr, "must not be null");
381 _to_region = new_to_region;
382 _compact_point = _to_region->bottom();
383 }
384
385 // Object fits into current region, record new location, if object does not move:
386 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
387 shenandoah_assert_not_forwarded(nullptr, p);
388 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
389 _preserved_marks->push_if_necessary(p, p->mark());
390 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
391 }
392 _compact_point += obj_size;
393 }
394 };
395
396 class ShenandoahPrepareForCompactionTask : public WorkerTask {
397 private:
398 PreservedMarksSet* const _preserved_marks;
399 ShenandoahHeap* const _heap;
400 ShenandoahHeapRegionSet** const _worker_slices;
401
402 public:
875 uint const _worker_id;
876
877 public:
878 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
879 _worker_id(worker_id) {}
880
881 void do_object(oop p) override {
882 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
883 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
884 size_t size = p->size();
885 if (FullGCForwarding::is_forwarded(p)) {
886 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
887 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
888 assert(compact_from != compact_to, "Forwarded object should move");
889 Copy::aligned_conjoint_words(compact_from, compact_to, size);
890 oop new_obj = cast_to_oop(compact_to);
891
892 // Restore the mark word before relativizing the stack chunk. The copy's
893 // mark word contains the full GC forwarding encoding, which would cause
894 // is_stackChunk() to read garbage (especially with compact headers).
895 new_obj->init_mark();
896 ContinuationGCSupport::relativize_stack_chunk(new_obj);
897 }
898 }
899 };
900
901 class ShenandoahCompactObjectsTask : public WorkerTask {
902 private:
903 ShenandoahHeap* const _heap;
904 ShenandoahHeapRegionSet** const _worker_slices;
905
906 public:
907 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
908 WorkerTask("Shenandoah Compact Objects"),
909 _heap(ShenandoahHeap::heap()),
910 _worker_slices(worker_slices) {
911 }
912
913 void work(uint worker_id) override {
914 ShenandoahParallelWorkerSession worker_session(worker_id);
915 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
1010 oop old_obj = cast_to_oop(r->bottom());
1011 if (!FullGCForwarding::is_forwarded(old_obj)) {
1012 // No need to move the object, it stays at the same slot
1013 continue;
1014 }
1015 size_t words_size = old_obj->size();
1016 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1017
1018 size_t old_start = r->index();
1019 size_t old_end = old_start + num_regions - 1;
1020 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1021 size_t new_end = new_start + num_regions - 1;
1022 assert(old_start != new_start, "must be real move");
1023 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1024
1025 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1026 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
1027 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1028
1029 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1030 new_obj->init_mark();
1031
1032 {
1033 ShenandoahAffiliation original_affiliation = r->affiliation();
1034 for (size_t c = old_start; c <= old_end; c++) {
1035 ShenandoahHeapRegion* r = heap->get_region(c);
1036 // Leave humongous region affiliation unchanged.
1037 r->make_regular_bypass();
1038 r->set_top(r->bottom());
1039 }
1040
1041 for (size_t c = new_start; c <= new_end; c++) {
1042 ShenandoahHeapRegion* r = heap->get_region(c);
1043 if (c == new_start) {
1044 r->make_humongous_start_bypass(original_affiliation);
1045 } else {
1046 r->make_humongous_cont_bypass(original_affiliation);
1047 }
1048
1049 // Trailing region may be non-full, record the remainder there
1050 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
|
234 // Coming out of Full GC, we would not have any forwarded objects.
235 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
236 heap->set_has_forwarded_objects(false);
237
238 heap->set_full_gc_move_in_progress(true);
239
240 // Setup workers for the rest
241 OrderAccess::fence();
242
243 // Initialize worker slices
244 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
245 for (uint i = 0; i < heap->max_workers(); i++) {
246 worker_slices[i] = new ShenandoahHeapRegionSet();
247 }
248
249 {
250 // The rest of code performs region moves, where region status is undefined
251 // until all phases run together.
252 ShenandoahHeapLocker lock(heap->lock());
253
254 FullGCForwarding::begin();
255
256 phase2_calculate_target_addresses(worker_slices);
257
258 OrderAccess::fence();
259
260 phase3_update_references();
261
262 phase4_compact_objects(worker_slices);
263
264 phase5_epilog();
265
266 FullGCForwarding::end();
267 }
268 heap->start_idle_span();
269
270 // Resize metaspace
271 MetaspaceGC::compute_new_size();
272
273 // Free worker slices
274 for (uint i = 0; i < heap->max_workers(); i++) {
275 delete worker_slices[i];
276 }
277 FREE_C_HEAP_ARRAY(worker_slices);
278
279 heap->set_full_gc_move_in_progress(false);
280 heap->set_full_gc_in_progress(false);
281
282 DEBUG_ONLY(heap->assert_no_self_forwards());
283
284 if (ShenandoahVerify) {
285 heap->verifier()->verify_after_fullgc(_generation);
286 }
349 void finish() {
350 assert(_to_region != nullptr, "should not happen");
351 _to_region->set_new_top(_compact_point);
352 }
353
354 bool is_compact_same_region() {
355 return _from_region == _to_region;
356 }
357
358 int empty_regions_pos() {
359 return _empty_regions_pos;
360 }
361
362 void do_object(oop p) override {
363 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
364 assert(_from_region != nullptr, "must set before work");
365 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
366 assert(_heap->marking_context()->is_marked(p), "must be marked");
367 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
368
369 size_t old_size = p->size();
370 size_t new_size = p->copy_size(old_size, p->mark());
371 size_t obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
372 if (_compact_point + obj_size > _to_region->end()) {
373 finish();
374
375 // Object doesn't fit. Pick next empty region and start compacting there.
376 ShenandoahHeapRegion* new_to_region;
377 if (_empty_regions_pos < _empty_regions.length()) {
378 new_to_region = _empty_regions.at(_empty_regions_pos);
379 _empty_regions_pos++;
380 } else {
381 // Out of empty region? Compact within the same region.
382 new_to_region = _from_region;
383 }
384
385 assert(new_to_region != _to_region, "must not reuse same to-region");
386 assert(new_to_region != nullptr, "must not be null");
387 _to_region = new_to_region;
388 _compact_point = _to_region->bottom();
389 obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
390 }
391
392 // Object fits into current region, record new location, if object does not move:
393 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
394 shenandoah_assert_not_forwarded(nullptr, p);
395 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
396 _preserved_marks->push_if_necessary(p, p->mark());
397 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
398 }
399 _compact_point += obj_size;
400 }
401 };
402
403 class ShenandoahPrepareForCompactionTask : public WorkerTask {
404 private:
405 PreservedMarksSet* const _preserved_marks;
406 ShenandoahHeap* const _heap;
407 ShenandoahHeapRegionSet** const _worker_slices;
408
409 public:
882 uint const _worker_id;
883
884 public:
885 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
886 _worker_id(worker_id) {}
887
888 void do_object(oop p) override {
889 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
890 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
891 size_t size = p->size();
892 if (FullGCForwarding::is_forwarded(p)) {
893 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
894 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
895 assert(compact_from != compact_to, "Forwarded object should move");
896 Copy::aligned_conjoint_words(compact_from, compact_to, size);
897 oop new_obj = cast_to_oop(compact_to);
898
899 // Restore the mark word before relativizing the stack chunk. The copy's
900 // mark word contains the full GC forwarding encoding, which would cause
901 // is_stackChunk() to read garbage (especially with compact headers).
902 new_obj->reinit_mark();
903 new_obj->initialize_hash_if_necessary(p);
904 ContinuationGCSupport::relativize_stack_chunk(new_obj);
905 }
906 }
907 };
908
909 class ShenandoahCompactObjectsTask : public WorkerTask {
910 private:
911 ShenandoahHeap* const _heap;
912 ShenandoahHeapRegionSet** const _worker_slices;
913
914 public:
915 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
916 WorkerTask("Shenandoah Compact Objects"),
917 _heap(ShenandoahHeap::heap()),
918 _worker_slices(worker_slices) {
919 }
920
921 void work(uint worker_id) override {
922 ShenandoahParallelWorkerSession worker_session(worker_id);
923 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
1018 oop old_obj = cast_to_oop(r->bottom());
1019 if (!FullGCForwarding::is_forwarded(old_obj)) {
1020 // No need to move the object, it stays at the same slot
1021 continue;
1022 }
1023 size_t words_size = old_obj->size();
1024 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
1025
1026 size_t old_start = r->index();
1027 size_t old_end = old_start + num_regions - 1;
1028 size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj));
1029 size_t new_end = new_start + num_regions - 1;
1030 assert(old_start != new_start, "must be real move");
1031 assert(r->is_stw_move_allowed(), "Region %zu should be movable", r->index());
1032
1033 log_debug(gc)("Full GC compaction moves humongous object from region %zu to region %zu", old_start, new_start);
1034 Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
1035 ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
1036
1037 oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
1038 new_obj->reinit_mark();
1039
1040 {
1041 ShenandoahAffiliation original_affiliation = r->affiliation();
1042 for (size_t c = old_start; c <= old_end; c++) {
1043 ShenandoahHeapRegion* r = heap->get_region(c);
1044 // Leave humongous region affiliation unchanged.
1045 r->make_regular_bypass();
1046 r->set_top(r->bottom());
1047 }
1048
1049 for (size_t c = new_start; c <= new_end; c++) {
1050 ShenandoahHeapRegion* r = heap->get_region(c);
1051 if (c == new_start) {
1052 r->make_humongous_start_bypass(original_affiliation);
1053 } else {
1054 r->make_humongous_cont_bypass(original_affiliation);
1055 }
1056
1057 // Trailing region may be non-full, record the remainder there
1058 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
|