228 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
229 heap->set_has_forwarded_objects(false);
230
231 heap->set_full_gc_move_in_progress(true);
232
233 // Setup workers for the rest
234 OrderAccess::fence();
235
236 // Initialize worker slices
237 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
238 for (uint i = 0; i < heap->max_workers(); i++) {
239 worker_slices[i] = new ShenandoahHeapRegionSet();
240 }
241
242 ShenandoahGenerationalHeap::TransferResult result;
243 {
244 // The rest of code performs region moves, where region status is undefined
245 // until all phases run together.
246 ShenandoahHeapLocker lock(heap->lock());
247
248 phase2_calculate_target_addresses(worker_slices);
249
250 OrderAccess::fence();
251
252 phase3_update_references();
253
254 phase4_compact_objects(worker_slices);
255
256 result = phase5_epilog();
257 }
258 if (heap->mode()->is_generational()) {
259 LogTarget(Info, gc, ergo) lt;
260 if (lt.is_enabled()) {
261 LogStream ls(lt);
262 result.print_on("Full GC", &ls);
263 }
264 }
265
266 // Resize metaspace
267 MetaspaceGC::compute_new_size();
268
269 // Free worker slices
270 for (uint i = 0; i < heap->max_workers(); i++) {
271 delete worker_slices[i];
272 }
273 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
274
275 heap->set_full_gc_move_in_progress(false);
276 heap->set_full_gc_in_progress(false);
341 }
342
343 void finish() {
344 assert(_to_region != nullptr, "should not happen");
345 _to_region->set_new_top(_compact_point);
346 }
347
348 bool is_compact_same_region() {
349 return _from_region == _to_region;
350 }
351
352 int empty_regions_pos() {
353 return _empty_regions_pos;
354 }
355
356 void do_object(oop p) {
357 assert(_from_region != nullptr, "must set before work");
358 assert(_heap->gc_generation()->complete_marking_context()->is_marked(p), "must be marked");
359 assert(!_heap->gc_generation()->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
360
361 size_t obj_size = p->size();
362 if (_compact_point + obj_size > _to_region->end()) {
363 finish();
364
365 // Object doesn't fit. Pick next empty region and start compacting there.
366 ShenandoahHeapRegion* new_to_region;
367 if (_empty_regions_pos < _empty_regions.length()) {
368 new_to_region = _empty_regions.at(_empty_regions_pos);
369 _empty_regions_pos++;
370 } else {
371 // Out of empty region? Compact within the same region.
372 new_to_region = _from_region;
373 }
374
375 assert(new_to_region != _to_region, "must not reuse same to-region");
376 assert(new_to_region != nullptr, "must not be null");
377 _to_region = new_to_region;
378 _compact_point = _to_region->bottom();
379 }
380
381 // Object fits into current region, record new location, if object does not move:
382 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
383 shenandoah_assert_not_forwarded(nullptr, p);
384 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
385 _preserved_marks->push_if_necessary(p, p->mark());
386 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
387 }
388 _compact_point += obj_size;
389 }
390 };
391
392 class ShenandoahPrepareForCompactionTask : public WorkerTask {
393 private:
394 PreservedMarksSet* const _preserved_marks;
395 ShenandoahHeap* const _heap;
396 ShenandoahHeapRegionSet** const _worker_slices;
397
398 public:
878 private:
879 ShenandoahHeap* const _heap;
880 uint const _worker_id;
881
882 public:
883 ShenandoahCompactObjectsClosure(uint worker_id) :
884 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
885
886 void do_object(oop p) {
887 assert(_heap->gc_generation()->complete_marking_context()->is_marked(p), "must be marked");
888 size_t size = p->size();
889 if (FullGCForwarding::is_forwarded(p)) {
890 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
891 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
892 assert(compact_from != compact_to, "Forwarded object should move");
893 Copy::aligned_conjoint_words(compact_from, compact_to, size);
894 oop new_obj = cast_to_oop(compact_to);
895
896 ContinuationGCSupport::relativize_stack_chunk(new_obj);
897 new_obj->init_mark();
898 }
899 }
900 };
901
902 class ShenandoahCompactObjectsTask : public WorkerTask {
903 private:
904 ShenandoahHeap* const _heap;
905 ShenandoahHeapRegionSet** const _worker_slices;
906
907 public:
908 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
909 WorkerTask("Shenandoah Compact Objects"),
910 _heap(ShenandoahHeap::heap()),
911 _worker_slices(worker_slices) {
912 }
913
914 void work(uint worker_id) {
915 ShenandoahParallelWorkerSession worker_session(worker_id);
916 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
917
|
228 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
229 heap->set_has_forwarded_objects(false);
230
231 heap->set_full_gc_move_in_progress(true);
232
233 // Setup workers for the rest
234 OrderAccess::fence();
235
236 // Initialize worker slices
237 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
238 for (uint i = 0; i < heap->max_workers(); i++) {
239 worker_slices[i] = new ShenandoahHeapRegionSet();
240 }
241
242 ShenandoahGenerationalHeap::TransferResult result;
243 {
244 // The rest of code performs region moves, where region status is undefined
245 // until all phases run together.
246 ShenandoahHeapLocker lock(heap->lock());
247
248 FullGCForwarding::begin();
249
250 phase2_calculate_target_addresses(worker_slices);
251
252 OrderAccess::fence();
253
254 phase3_update_references();
255
256 phase4_compact_objects(worker_slices);
257
258 result = phase5_epilog();
259
260 FullGCForwarding::end();
261 }
262 if (heap->mode()->is_generational()) {
263 LogTarget(Info, gc, ergo) lt;
264 if (lt.is_enabled()) {
265 LogStream ls(lt);
266 result.print_on("Full GC", &ls);
267 }
268 }
269
270 // Resize metaspace
271 MetaspaceGC::compute_new_size();
272
273 // Free worker slices
274 for (uint i = 0; i < heap->max_workers(); i++) {
275 delete worker_slices[i];
276 }
277 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
278
279 heap->set_full_gc_move_in_progress(false);
280 heap->set_full_gc_in_progress(false);
345 }
346
347 void finish() {
348 assert(_to_region != nullptr, "should not happen");
349 _to_region->set_new_top(_compact_point);
350 }
351
352 bool is_compact_same_region() {
353 return _from_region == _to_region;
354 }
355
356 int empty_regions_pos() {
357 return _empty_regions_pos;
358 }
359
360 void do_object(oop p) {
361 assert(_from_region != nullptr, "must set before work");
362 assert(_heap->gc_generation()->complete_marking_context()->is_marked(p), "must be marked");
363 assert(!_heap->gc_generation()->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
364
365 size_t old_size = p->size();
366 size_t new_size = p->copy_size(old_size, p->mark());
367 size_t obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
368 if (_compact_point + obj_size > _to_region->end()) {
369 finish();
370
371 // Object doesn't fit. Pick next empty region and start compacting there.
372 ShenandoahHeapRegion* new_to_region;
373 if (_empty_regions_pos < _empty_regions.length()) {
374 new_to_region = _empty_regions.at(_empty_regions_pos);
375 _empty_regions_pos++;
376 } else {
377 // Out of empty region? Compact within the same region.
378 new_to_region = _from_region;
379 }
380
381 assert(new_to_region != _to_region, "must not reuse same to-region");
382 assert(new_to_region != nullptr, "must not be null");
383 _to_region = new_to_region;
384 _compact_point = _to_region->bottom();
385 obj_size = _compact_point == cast_from_oop<HeapWord*>(p) ? old_size : new_size;
386 }
387
388 // Object fits into current region, record new location, if object does not move:
389 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
390 shenandoah_assert_not_forwarded(nullptr, p);
391 if (_compact_point != cast_from_oop<HeapWord*>(p)) {
392 _preserved_marks->push_if_necessary(p, p->mark());
393 FullGCForwarding::forward_to(p, cast_to_oop(_compact_point));
394 }
395 _compact_point += obj_size;
396 }
397 };
398
399 class ShenandoahPrepareForCompactionTask : public WorkerTask {
400 private:
401 PreservedMarksSet* const _preserved_marks;
402 ShenandoahHeap* const _heap;
403 ShenandoahHeapRegionSet** const _worker_slices;
404
405 public:
885 private:
886 ShenandoahHeap* const _heap;
887 uint const _worker_id;
888
889 public:
890 ShenandoahCompactObjectsClosure(uint worker_id) :
891 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
892
893 void do_object(oop p) {
894 assert(_heap->gc_generation()->complete_marking_context()->is_marked(p), "must be marked");
895 size_t size = p->size();
896 if (FullGCForwarding::is_forwarded(p)) {
897 HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
898 HeapWord* compact_to = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(p));
899 assert(compact_from != compact_to, "Forwarded object should move");
900 Copy::aligned_conjoint_words(compact_from, compact_to, size);
901 oop new_obj = cast_to_oop(compact_to);
902
903 ContinuationGCSupport::relativize_stack_chunk(new_obj);
904 new_obj->init_mark();
905 new_obj->initialize_hash_if_necessary(p);
906 }
907 }
908 };
909
910 class ShenandoahCompactObjectsTask : public WorkerTask {
911 private:
912 ShenandoahHeap* const _heap;
913 ShenandoahHeapRegionSet** const _worker_slices;
914
915 public:
916 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
917 WorkerTask("Shenandoah Compact Objects"),
918 _heap(ShenandoahHeap::heap()),
919 _worker_slices(worker_slices) {
920 }
921
922 void work(uint worker_id) {
923 ShenandoahParallelWorkerSession worker_session(worker_id);
924 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
925
|