225 // Coming out of Full GC, we would not have any forwarded objects.
226 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
227 heap->set_has_forwarded_objects(false);
228
229 heap->set_full_gc_move_in_progress(true);
230
231 // Setup workers for the rest
232 OrderAccess::fence();
233
234 // Initialize worker slices
235 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
236 for (uint i = 0; i < heap->max_workers(); i++) {
237 worker_slices[i] = new ShenandoahHeapRegionSet();
238 }
239
240 {
241 // The rest of code performs region moves, where region status is undefined
242 // until all phases run together.
243 ShenandoahHeapLocker lock(heap->lock());
244
245 phase2_calculate_target_addresses(worker_slices);
246
247 OrderAccess::fence();
248
249 phase3_update_references();
250
251 phase4_compact_objects(worker_slices);
252
253 phase5_epilog();
254 }
255
256 // Resize metaspace
257 MetaspaceGC::compute_new_size();
258
259 // Free worker slices
260 for (uint i = 0; i < heap->max_workers(); i++) {
261 delete worker_slices[i];
262 }
263 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
264
265 heap->set_full_gc_move_in_progress(false);
266 heap->set_full_gc_in_progress(false);
267
268 if (ShenandoahVerify) {
269 heap->verifier()->verify_after_fullgc(_generation);
270 }
271
272 if (VerifyAfterGC) {
273 Universe::verify();
333 void finish() {
334 assert(_to_region != nullptr, "should not happen");
335 _to_region->set_new_top(_compact_point);
336 }
337
338 bool is_compact_same_region() {
339 return _from_region == _to_region;
340 }
341
342 int empty_regions_pos() {
343 return _empty_regions_pos;
344 }
345
346 void do_object(oop p) override {
347 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
348 assert(_from_region != nullptr, "must set before work");
349 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
350 assert(_heap->marking_context()->is_marked(p), "must be marked");
351 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
352
353 size_t obj_size = p->size();
354 if (_compact_point + obj_size > _to_region->end()) {
355 finish();
356
357 // Object doesn't fit. Pick next empty region and start compacting there.
358 ShenandoahHeapRegion* new_to_region;
359 if (_empty_regions_pos < _empty_regions.length()) {
360 new_to_region = _empty_regions.at(_empty_regions_pos);
361 _empty_regions_pos++;
362 } else {
363 // Out of empty region? Compact within the same region.
364 new_to_region = _from_region;
365 }
366
367 assert(new_to_region != _to_region, "must not reuse same to-region");
368 assert(new_to_region != nullptr, "must not be null");
369 _to_region = new_to_region;
370 _compact_point = _to_region->bottom();
371 }
372
373 // Object fits into current region, record new location, if object does not move:
374 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
375 shenandoah_assert_not_forwarded(nullptr, p);
376 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
377 _preserved_marks->push_if_necessary(p, p->mark());
378 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
379 }
380 _compact_point += obj_size;
381 }
382 };
383
384 class ShenandoahPrepareForCompactionTask : public WorkerTask {
385 private:
386 PreservedMarksSet* const _preserved_marks;
387 ShenandoahHeap* const _heap;
388 ShenandoahHeapRegionSet** const _worker_slices;
389
390 public:
861 private:
862 uint const _worker_id;
863
864 public:
865 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
866 _worker_id(worker_id) {}
867
868 void do_object(oop p) override {
869 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
870 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
871 size_t size = p->size();
872 if (FullGCForwarding::is_forwarded(p)) {
873 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
874 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
875 assert(compact_from != compact_to, "Forwarded object should move");
876 Copy::aligned_conjoint_words(compact_from, compact_to, size);
877 oop new_obj = cast_to_oop(compact_to);
878
879 ContinuationGCSupport::relativize_stack_chunk(new_obj);
880 new_obj->init_mark();
881 }
882 }
883 };
884
885 class ShenandoahCompactObjectsTask : public WorkerTask {
886 private:
887 ShenandoahHeap* const _heap;
888 ShenandoahHeapRegionSet** const _worker_slices;
889
890 public:
891 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
892 WorkerTask("Shenandoah Compact Objects"),
893 _heap(ShenandoahHeap::heap()),
894 _worker_slices(worker_slices) {
895 }
896
897 void work(uint worker_id) override {
898 ShenandoahParallelWorkerSession worker_session(worker_id);
899 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
900
|
225 // Coming out of Full GC, we would not have any forwarded objects.
226 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
227 heap->set_has_forwarded_objects(false);
228
229 heap->set_full_gc_move_in_progress(true);
230
231 // Setup workers for the rest
232 OrderAccess::fence();
233
234 // Initialize worker slices
235 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
236 for (uint i = 0; i < heap->max_workers(); i++) {
237 worker_slices[i] = new ShenandoahHeapRegionSet();
238 }
239
240 {
241 // The rest of code performs region moves, where region status is undefined
242 // until all phases run together.
243 ShenandoahHeapLocker lock(heap->lock());
244
245 FullGCForwarding::begin();
246
247 phase2_calculate_target_addresses(worker_slices);
248
249 OrderAccess::fence();
250
251 phase3_update_references();
252
253 phase4_compact_objects(worker_slices);
254
255 phase5_epilog();
256
257 FullGCForwarding::end();
258 }
259
260 // Resize metaspace
261 MetaspaceGC::compute_new_size();
262
263 // Free worker slices
264 for (uint i = 0; i < heap->max_workers(); i++) {
265 delete worker_slices[i];
266 }
267 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
268
269 heap->set_full_gc_move_in_progress(false);
270 heap->set_full_gc_in_progress(false);
271
272 if (ShenandoahVerify) {
273 heap->verifier()->verify_after_fullgc(_generation);
274 }
275
276 if (VerifyAfterGC) {
277 Universe::verify();
337 void finish() {
338 assert(_to_region != nullptr, "should not happen");
339 _to_region->set_new_top(_compact_point);
340 }
341
342 bool is_compact_same_region() {
343 return _from_region == _to_region;
344 }
345
346 int empty_regions_pos() {
347 return _empty_regions_pos;
348 }
349
350 void do_object(oop p) override {
351 shenandoah_assert_mark_complete(cast_from_oop<HeapWord*>(p));
352 assert(_from_region != nullptr, "must set before work");
353 assert(_heap->global_generation()->is_mark_complete(), "marking must be finished");
354 assert(_heap->marking_context()->is_marked(p), "must be marked");
355 assert(!_heap->marking_context()->allocated_after_mark_start(p), "must be truly marked");
356
357 size_t old_size = p->size();
358 size_t new_size = p->copy_size(old_size, p->mark());
359 size_t obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
360 if (_compact_point + obj_size > _to_region->end()) {
361 finish();
362
363 // Object doesn't fit. Pick next empty region and start compacting there.
364 ShenandoahHeapRegion* new_to_region;
365 if (_empty_regions_pos < _empty_regions.length()) {
366 new_to_region = _empty_regions.at(_empty_regions_pos);
367 _empty_regions_pos++;
368 } else {
369 // Out of empty region? Compact within the same region.
370 new_to_region = _from_region;
371 }
372
373 assert(new_to_region != _to_region, "must not reuse same to-region");
374 assert(new_to_region != nullptr, "must not be null");
375 _to_region = new_to_region;
376 _compact_point = _to_region->bottom();
377 obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
378 }
379
380 // Object fits into current region, record new location, if object does not move:
381 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
382 shenandoah_assert_not_forwarded(nullptr, p);
383 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
384 _preserved_marks->push_if_necessary(p, p->mark());
385 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
386 }
387 _compact_point += obj_size;
388 }
389 };
390
391 class ShenandoahPrepareForCompactionTask : public WorkerTask {
392 private:
393 PreservedMarksSet* const _preserved_marks;
394 ShenandoahHeap* const _heap;
395 ShenandoahHeapRegionSet** const _worker_slices;
396
397 public:
868 private:
869 uint const _worker_id;
870
871 public:
872 explicit ShenandoahCompactObjectsClosure(uint worker_id) :
873 _worker_id(worker_id) {}
874
875 void do_object(oop p) override {
876 assert(ShenandoahHeap::heap()->global_generation()->is_mark_complete(), "marking must be finished");
877 assert(ShenandoahHeap::heap()->marking_context()->is_marked(p), "must be marked");
878 size_t size = p->size();
879 if (FullGCForwarding::is_forwarded(p)) {
880 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
881 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
882 assert(compact_from != compact_to, "Forwarded object should move");
883 Copy::aligned_conjoint_words(compact_from, compact_to, size);
884 oop new_obj = cast_to_oop(compact_to);
885
886 ContinuationGCSupport::relativize_stack_chunk(new_obj);
887 new_obj->init_mark();
888 new_obj->initialize_hash_if_necessary(p);
889 }
890 }
891 };
892
893 class ShenandoahCompactObjectsTask : public WorkerTask {
894 private:
895 ShenandoahHeap* const _heap;
896 ShenandoahHeapRegionSet** const _worker_slices;
897
898 public:
899 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
900 WorkerTask("Shenandoah Compact Objects"),
901 _heap(ShenandoahHeap::heap()),
902 _worker_slices(worker_slices) {
903 }
904
905 void work(uint worker_id) override {
906 ShenandoahParallelWorkerSession worker_session(worker_id);
907 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
908
|