61 #include "gc/shared/oopStorageSetParState.inline.hpp"
62 #include "gc/shared/parallelCleaning.hpp"
63 #include "gc/shared/preservedMarks.inline.hpp"
64 #include "gc/shared/referencePolicy.hpp"
65 #include "gc/shared/referenceProcessor.hpp"
66 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
67 #include "gc/shared/spaceDecorator.hpp"
68 #include "gc/shared/taskTerminator.hpp"
69 #include "gc/shared/weakProcessor.inline.hpp"
70 #include "gc/shared/workerPolicy.hpp"
71 #include "gc/shared/workerThread.hpp"
72 #include "gc/shared/workerUtils.hpp"
73 #include "logging/log.hpp"
74 #include "memory/iterator.inline.hpp"
75 #include "memory/memoryReserver.hpp"
76 #include "memory/metaspaceUtils.hpp"
77 #include "memory/resourceArea.hpp"
78 #include "memory/universe.hpp"
79 #include "nmt/memTracker.hpp"
80 #include "oops/access.inline.hpp"
81 #include "oops/instanceClassLoaderKlass.inline.hpp"
82 #include "oops/instanceKlass.inline.hpp"
83 #include "oops/instanceMirrorKlass.inline.hpp"
84 #include "oops/methodData.hpp"
85 #include "oops/objArrayKlass.inline.hpp"
86 #include "oops/oop.inline.hpp"
87 #include "runtime/handles.inline.hpp"
88 #include "runtime/java.hpp"
89 #include "runtime/safepoint.hpp"
90 #include "runtime/threads.hpp"
91 #include "runtime/vmThread.hpp"
92 #include "services/memoryService.hpp"
93 #include "utilities/align.hpp"
94 #include "utilities/debug.hpp"
95 #include "utilities/events.hpp"
96 #include "utilities/formatBuffer.hpp"
97 #include "utilities/macros.hpp"
98 #include "utilities/stack.inline.hpp"
99 #if INCLUDE_JVMCI
100 #include "jvmci/jvmci.hpp"
101 #endif
102
103 #include <math.h>
104
105 // All sizes are in HeapWords.
106 const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words
107 const size_t ParallelCompactData::RegionSize = (size_t)1 << Log2RegionSize;
108 static_assert(ParallelCompactData::RegionSize >= BitsPerWord, "region-start bit word-aligned");
109 const size_t ParallelCompactData::RegionSizeBytes =
110 RegionSize << LogHeapWordSize;
111 const size_t ParallelCompactData::RegionSizeOffsetMask = RegionSize - 1;
112 const size_t ParallelCompactData::RegionAddrOffsetMask = RegionSizeBytes - 1;
113 const size_t ParallelCompactData::RegionAddrMask = ~RegionAddrOffsetMask;
114
115 const ParallelCompactData::RegionData::region_sz_t
116 ParallelCompactData::RegionData::dc_shift = 27;
117
118 const ParallelCompactData::RegionData::region_sz_t
119 ParallelCompactData::RegionData::dc_mask = ~0U << dc_shift;
120
121 const ParallelCompactData::RegionData::region_sz_t
684 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
685 bool eden_empty = eden_space->is_empty();
686
687 // Update heap occupancy information which is used as input to the soft ref
688 // clearing policy at the next gc.
689 Universe::heap()->update_capacity_and_used_at_gc();
690
691 bool young_gen_empty = eden_empty && from_space->is_empty() &&
692 to_space->is_empty();
693
694 PSCardTable* ct = heap->card_table();
695 MemRegion old_mr = heap->old_gen()->committed();
696 if (young_gen_empty) {
697 ct->clear_MemRegion(old_mr);
698 } else {
699 ct->dirty_MemRegion(old_mr);
700 }
701
702 heap->prune_scavengable_nmethods();
703
704 #if COMPILER2_OR_JVMCI
705 DerivedPointerTable::update_pointers();
706 #endif
707
708 // Signal that we have completed a visit to all live objects.
709 Universe::heap()->record_whole_heap_examined_timestamp();
710 }
711
712 HeapWord* PSParallelCompact::compute_dense_prefix_for_old_space(MutableSpace* old_space,
713 HeapWord* full_region_prefix_end) {
714 const size_t region_size = ParallelCompactData::RegionSize;
715 const ParallelCompactData& sd = summary_data();
716
717 // Iteration starts with the region *after* the full-region-prefix-end.
718 const RegionData* const start_region = sd.addr_to_region_ptr(full_region_prefix_end);
719 // If final region is not full, iteration stops before that region,
720 // because fill_dense_prefix_end assumes that prefix_end <= top.
721 const RegionData* const end_region = sd.addr_to_region_ptr(old_space->top());
722 assert(start_region <= end_region, "inv");
723
724 size_t max_waste = old_space->capacity_in_words() * (MarkSweepDeadRatio / 100.0);
725 const RegionData* cur_region = start_region;
726 for (/* empty */; cur_region < end_region; ++cur_region) {
960 WorkerPolicy::calc_active_workers(ParallelScavengeHeap::heap()->workers().max_workers(),
961 ParallelScavengeHeap::heap()->workers().active_workers(),
962 Threads::number_of_non_daemon_threads());
963 ParallelScavengeHeap::heap()->workers().set_active_workers(active_workers);
964
965 GCTraceCPUTime tcpu(&_gc_tracer);
966 GCTraceTime(Info, gc) tm("Pause Full", nullptr, gc_cause, true);
967
968 heap->pre_full_gc_dump(&_gc_timer);
969
970 TraceCollectorStats tcs(counters());
971 TraceMemoryManagerStats tms(heap->old_gc_manager(), gc_cause, "end of major GC");
972
973 if (log_is_enabled(Debug, gc, heap, exit)) {
974 accumulated_time()->start();
975 }
976
977 // Let the size policy know we're starting
978 size_policy->major_collection_begin();
979
980 #if COMPILER2_OR_JVMCI
981 DerivedPointerTable::clear();
982 #endif
983
984 ref_processor()->start_discovery(clear_all_soft_refs);
985
986 marking_phase(&_gc_tracer);
987
988 summary_phase(should_do_max_compaction);
989
990 #if COMPILER2_OR_JVMCI
991 assert(DerivedPointerTable::is_active(), "Sanity");
992 DerivedPointerTable::set_active(false);
993 #endif
994
995 forward_to_new_addr();
996
997 adjust_pointers();
998
999 compact();
1000
1001 ParCompactionManager::_preserved_marks_set->restore(&ParallelScavengeHeap::heap()->workers());
1002
1003 ParCompactionManager::verify_all_region_stack_empty();
1004
1005 // Reset the mark bitmap, summary data, and do other bookkeeping. Must be
1006 // done before resizing.
1007 post_compact();
1008
1009 size_policy->major_collection_end();
1010
1011 size_policy->sample_old_gen_used_bytes(MAX2(pre_gc_values.old_gen_used(), old_gen->used_in_bytes()));
1012
1013 if (UseAdaptiveSizePolicy) {
1154 for (uint i = 0; i < num_workers; ++i) {
1155 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(i);
1156 cm->flush_and_destroy_marking_stats_cache();
1157 }
1158 }
1159
1160 class PSParallelCleaningTask : public WorkerTask {
1161 bool _unloading_occurred;
1162 CodeCacheUnloadingTask _code_cache_task;
1163 // Prune dead klasses from subklass/sibling/implementor lists.
1164 KlassCleaningTask _klass_cleaning_task;
1165
1166 public:
1167 PSParallelCleaningTask(bool unloading_occurred) :
1168 WorkerTask("PS Parallel Cleaning"),
1169 _unloading_occurred(unloading_occurred),
1170 _code_cache_task(unloading_occurred),
1171 _klass_cleaning_task() {}
1172
1173 void work(uint worker_id) {
1174 #if INCLUDE_JVMCI
1175 if (EnableJVMCI && worker_id == 0) {
1176 // Serial work; only first worker.
1177 // Clean JVMCI metadata handles.
1178 JVMCI::do_unloading(_unloading_occurred);
1179 }
1180 #endif
1181
1182 // Do first pass of code cache cleaning.
1183 _code_cache_task.work(worker_id);
1184
1185 // Clean all klasses that were not unloaded.
1186 // The weak metadata in klass doesn't need to be
1187 // processed if there was no unloading.
1188 if (_unloading_occurred) {
1189 _klass_cleaning_task.work();
1190 }
1191 }
1192 };
1193
1194 void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
1195 // Recursively traverse all live objects and mark them
1196 GCTraceTime(Info, gc, phases) tm("Marking Phase", &_gc_timer);
1197
1198 uint active_gc_threads = ParallelScavengeHeap::heap()->workers().active_workers();
1199
1200 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_mark);
1201 {
1428
1429 // Split [start, end) evenly for a number of workers and return the
1430 // range for worker_id.
1431 static void split_regions_for_worker(size_t start, size_t end,
1432 uint worker_id, uint num_workers,
1433 size_t* worker_start, size_t* worker_end) {
1434 assert(start < end, "precondition");
1435 assert(num_workers > 0, "precondition");
1436 assert(worker_id < num_workers, "precondition");
1437
1438 size_t num_regions = end - start;
1439 size_t num_regions_per_worker = num_regions / num_workers;
1440 size_t remainder = num_regions % num_workers;
1441 // The first few workers will get one extra.
1442 *worker_start = start + worker_id * num_regions_per_worker
1443 + MIN2(checked_cast<size_t>(worker_id), remainder);
1444 *worker_end = *worker_start + num_regions_per_worker
1445 + (worker_id < remainder ? 1 : 0);
1446 }
1447
1448 void PSParallelCompact::forward_to_new_addr() {
1449 GCTraceTime(Info, gc, phases) tm("Forward", &_gc_timer);
1450 uint nworkers = ParallelScavengeHeap::heap()->workers().active_workers();
1451
1452 struct ForwardTask final : public WorkerTask {
1453 uint _num_workers;
1454
1455 explicit ForwardTask(uint num_workers) :
1456 WorkerTask("PSForward task"),
1457 _num_workers(num_workers) {}
1458
1459 static void forward_objs_in_range(ParCompactionManager* cm,
1460 HeapWord* start,
1461 HeapWord* end,
1462 HeapWord* destination) {
1463 HeapWord* cur_addr = start;
1464 HeapWord* new_addr = destination;
1465
1466 while (cur_addr < end) {
1467 cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end);
1468 if (cur_addr >= end) {
1469 return;
1470 }
1471 assert(mark_bitmap()->is_marked(cur_addr), "inv");
1472 oop obj = cast_to_oop(cur_addr);
1473 if (new_addr != cur_addr) {
1474 cm->preserved_marks()->push_if_necessary(obj, obj->mark());
1475 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
1476 }
1477 size_t obj_size = obj->size();
1478 new_addr += obj_size;
1479 cur_addr += obj_size;
1480 }
1481 }
1482
1483 void work(uint worker_id) override {
1484 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
1485 for (uint id = old_space_id; id < last_space_id; ++id) {
1486 MutableSpace* sp = PSParallelCompact::space(SpaceId(id));
1487 HeapWord* dense_prefix_addr = dense_prefix(SpaceId(id));
1488 HeapWord* top = sp->top();
1489
1490 if (dense_prefix_addr == top) {
1491 // Empty space
1492 continue;
1493 }
1494
2095
2096 HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) {
2097 ParallelCompactData& sd = summary_data();
2098 assert(sd.is_region_aligned(region_start_addr), "precondition");
2099
2100 // Use per-region partial_obj_size to locate the end of the obj, that extends
2101 // to region_start_addr.
2102 size_t start_region_idx = sd.addr_to_region_idx(region_start_addr);
2103 size_t end_region_idx = sd.region_count();
2104 size_t accumulated_size = 0;
2105 for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) {
2106 size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size();
2107 accumulated_size += cur_partial_obj_size;
2108 if (cur_partial_obj_size != ParallelCompactData::RegionSize) {
2109 break;
2110 }
2111 }
2112 return region_start_addr + accumulated_size;
2113 }
2114
2115 // Use region_idx as the destination region, and evacuate all live objs on its
2116 // source regions to this destination region.
2117 void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx)
2118 {
2119 ParMarkBitMap* const bitmap = mark_bitmap();
2120 ParallelCompactData& sd = summary_data();
2121 RegionData* const region_ptr = sd.region(region_idx);
2122
2123 // Get the source region and related info.
2124 size_t src_region_idx = region_ptr->source_region();
2125 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx));
2126 HeapWord* src_space_top = _space_info[src_space_id].space()->top();
2127 HeapWord* dest_addr = sd.region_to_addr(region_idx);
2128
2129 closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx));
2130
2131 // Adjust src_region_idx to prepare for decrementing destination counts (the
2132 // destination count is not decremented when a region is copied to itself).
2133 if (src_region_idx == region_idx) {
2134 src_region_idx += 1;
2210 HeapWord* cur_addr = closure.source();
2211 HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1),
2212 src_space_top);
2213 // To handle the case where the final obj in source region extends to next region.
2214 HeapWord* final_obj_start = (end_addr == src_space_top)
2215 ? nullptr
2216 : sd.addr_to_region_ptr(end_addr)->partial_obj_addr();
2217 // Apply closure on objs inside [cur_addr, end_addr)
2218 do {
2219 cur_addr = bitmap->find_obj_beg(cur_addr, end_addr);
2220 if (cur_addr == end_addr) {
2221 break;
2222 }
2223 size_t obj_size;
2224 if (final_obj_start == cur_addr) {
2225 obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr);
2226 } else {
2227 // This obj doesn't extend into next region; size() is safe to use.
2228 obj_size = cast_to_oop(cur_addr)->size();
2229 }
2230 closure.do_addr(cur_addr, obj_size);
2231 cur_addr += obj_size;
2232 } while (cur_addr < end_addr && !closure.is_full());
2233
2234 if (closure.is_full()) {
2235 decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source());
2236 closure.complete_region(dest_addr, region_ptr);
2237 return;
2238 }
2239
2240 decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr);
2241
2242 // Move to the next source region, possibly switching spaces as well. All
2243 // args except end_addr may be modified.
2244 src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr);
2245 } while (true);
2246 }
2247
2248 void PSParallelCompact::fill_and_update_region(ParCompactionManager* cm, size_t region_idx)
2249 {
2250 MoveAndUpdateClosure cl(mark_bitmap(), region_idx);
2330 }
2331
2332 void MoveAndUpdateClosure::copy_partial_obj(size_t partial_obj_size)
2333 {
2334 size_t words = MIN2(partial_obj_size, words_remaining());
2335
2336 // This test is necessary; if omitted, the pointer updates to a partial object
2337 // that crosses the dense prefix boundary could be overwritten.
2338 if (source() != copy_destination()) {
2339 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2340 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2341 }
2342 update_state(words);
2343 }
2344
2345 void MoveAndUpdateClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2346 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::NormalRegion, "Region should be finished");
2347 region_ptr->set_completed();
2348 }
2349
2350 void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) {
2351 assert(destination() != nullptr, "sanity");
2352 _source = addr;
2353
2354 // The start_array must be updated even if the object is not moving.
2355 if (_start_array != nullptr) {
2356 _start_array->update_for_block(destination(), destination() + words);
2357 }
2358
2359 // Avoid overflow
2360 words = MIN2(words, words_remaining());
2361 assert(words > 0, "inv");
2362
2363 if (copy_destination() != source()) {
2364 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2365 assert(source() != destination(), "inv");
2366 assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
2367 assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
2368 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2369 cast_to_oop(copy_destination())->init_mark();
2370 }
2371
2372 update_state(words);
2373 }
2374
2375 void MoveAndUpdateShadowClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2376 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::ShadowRegion, "Region should be shadow");
2377 // Record the shadow region index
2378 region_ptr->set_shadow_region(_shadow);
2379 // Mark the shadow region as filled to indicate the data is ready to be
2380 // copied back
2381 region_ptr->mark_filled();
2382 // Try to copy the content of the shadow region back to its corresponding
2383 // heap region if available; the GC thread that decreases the destination
2384 // count to zero will do the copying otherwise (see
2385 // PSParallelCompact::decrement_destination_counts).
2386 if (((region_ptr->available() && region_ptr->claim()) || region_ptr->claimed()) && region_ptr->mark_copied()) {
2387 region_ptr->set_completed();
2388 PSParallelCompact::copy_back(PSParallelCompact::summary_data().region_to_addr(_shadow), dest_addr);
2389 ParCompactionManager::push_shadow_region_mt_safe(_shadow);
|
61 #include "gc/shared/oopStorageSetParState.inline.hpp"
62 #include "gc/shared/parallelCleaning.hpp"
63 #include "gc/shared/preservedMarks.inline.hpp"
64 #include "gc/shared/referencePolicy.hpp"
65 #include "gc/shared/referenceProcessor.hpp"
66 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
67 #include "gc/shared/spaceDecorator.hpp"
68 #include "gc/shared/taskTerminator.hpp"
69 #include "gc/shared/weakProcessor.inline.hpp"
70 #include "gc/shared/workerPolicy.hpp"
71 #include "gc/shared/workerThread.hpp"
72 #include "gc/shared/workerUtils.hpp"
73 #include "logging/log.hpp"
74 #include "memory/iterator.inline.hpp"
75 #include "memory/memoryReserver.hpp"
76 #include "memory/metaspaceUtils.hpp"
77 #include "memory/resourceArea.hpp"
78 #include "memory/universe.hpp"
79 #include "nmt/memTracker.hpp"
80 #include "oops/access.inline.hpp"
81 #include "oops/flatArrayKlass.inline.hpp"
82 #include "oops/instanceClassLoaderKlass.inline.hpp"
83 #include "oops/instanceKlass.inline.hpp"
84 #include "oops/instanceMirrorKlass.inline.hpp"
85 #include "oops/methodData.hpp"
86 #include "oops/objArrayKlass.inline.hpp"
87 #include "oops/oop.inline.hpp"
88 #include "runtime/arguments.hpp"
89 #include "runtime/handles.inline.hpp"
90 #include "runtime/java.hpp"
91 #include "runtime/safepoint.hpp"
92 #include "runtime/threads.hpp"
93 #include "runtime/vmThread.hpp"
94 #include "services/memoryService.hpp"
95 #include "utilities/align.hpp"
96 #include "utilities/debug.hpp"
97 #include "utilities/events.hpp"
98 #include "utilities/formatBuffer.hpp"
99 #include "utilities/macros.hpp"
100 #include "utilities/stack.inline.hpp"
101
102 #include <math.h>
103
104 // All sizes are in HeapWords.
105 const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words
106 const size_t ParallelCompactData::RegionSize = (size_t)1 << Log2RegionSize;
107 static_assert(ParallelCompactData::RegionSize >= BitsPerWord, "region-start bit word-aligned");
108 const size_t ParallelCompactData::RegionSizeBytes =
109 RegionSize << LogHeapWordSize;
110 const size_t ParallelCompactData::RegionSizeOffsetMask = RegionSize - 1;
111 const size_t ParallelCompactData::RegionAddrOffsetMask = RegionSizeBytes - 1;
112 const size_t ParallelCompactData::RegionAddrMask = ~RegionAddrOffsetMask;
113
114 const ParallelCompactData::RegionData::region_sz_t
115 ParallelCompactData::RegionData::dc_shift = 27;
116
117 const ParallelCompactData::RegionData::region_sz_t
118 ParallelCompactData::RegionData::dc_mask = ~0U << dc_shift;
119
120 const ParallelCompactData::RegionData::region_sz_t
683 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
684 bool eden_empty = eden_space->is_empty();
685
686 // Update heap occupancy information which is used as input to the soft ref
687 // clearing policy at the next gc.
688 Universe::heap()->update_capacity_and_used_at_gc();
689
690 bool young_gen_empty = eden_empty && from_space->is_empty() &&
691 to_space->is_empty();
692
693 PSCardTable* ct = heap->card_table();
694 MemRegion old_mr = heap->old_gen()->committed();
695 if (young_gen_empty) {
696 ct->clear_MemRegion(old_mr);
697 } else {
698 ct->dirty_MemRegion(old_mr);
699 }
700
701 heap->prune_scavengable_nmethods();
702
703 #ifdef COMPILER2
704 DerivedPointerTable::update_pointers();
705 #endif // COMPILER2
706
707 // Signal that we have completed a visit to all live objects.
708 Universe::heap()->record_whole_heap_examined_timestamp();
709 }
710
711 HeapWord* PSParallelCompact::compute_dense_prefix_for_old_space(MutableSpace* old_space,
712 HeapWord* full_region_prefix_end) {
713 const size_t region_size = ParallelCompactData::RegionSize;
714 const ParallelCompactData& sd = summary_data();
715
716 // Iteration starts with the region *after* the full-region-prefix-end.
717 const RegionData* const start_region = sd.addr_to_region_ptr(full_region_prefix_end);
718 // If final region is not full, iteration stops before that region,
719 // because fill_dense_prefix_end assumes that prefix_end <= top.
720 const RegionData* const end_region = sd.addr_to_region_ptr(old_space->top());
721 assert(start_region <= end_region, "inv");
722
723 size_t max_waste = old_space->capacity_in_words() * (MarkSweepDeadRatio / 100.0);
724 const RegionData* cur_region = start_region;
725 for (/* empty */; cur_region < end_region; ++cur_region) {
959 WorkerPolicy::calc_active_workers(ParallelScavengeHeap::heap()->workers().max_workers(),
960 ParallelScavengeHeap::heap()->workers().active_workers(),
961 Threads::number_of_non_daemon_threads());
962 ParallelScavengeHeap::heap()->workers().set_active_workers(active_workers);
963
964 GCTraceCPUTime tcpu(&_gc_tracer);
965 GCTraceTime(Info, gc) tm("Pause Full", nullptr, gc_cause, true);
966
967 heap->pre_full_gc_dump(&_gc_timer);
968
969 TraceCollectorStats tcs(counters());
970 TraceMemoryManagerStats tms(heap->old_gc_manager(), gc_cause, "end of major GC");
971
972 if (log_is_enabled(Debug, gc, heap, exit)) {
973 accumulated_time()->start();
974 }
975
976 // Let the size policy know we're starting
977 size_policy->major_collection_begin();
978
979 #ifdef COMPILER2
980 DerivedPointerTable::clear();
981 #endif // COMPILER2
982
983 ref_processor()->start_discovery(clear_all_soft_refs);
984
985 marking_phase(&_gc_tracer);
986
987 summary_phase(should_do_max_compaction);
988
989 #ifdef COMPILER2
990 assert(DerivedPointerTable::is_active(), "Sanity");
991 DerivedPointerTable::set_active(false);
992 #endif // COMPILER2
993
994 forward_to_new_addr();
995
996 adjust_pointers();
997
998 compact();
999
1000 ParCompactionManager::_preserved_marks_set->restore(&ParallelScavengeHeap::heap()->workers());
1001
1002 ParCompactionManager::verify_all_region_stack_empty();
1003
1004 // Reset the mark bitmap, summary data, and do other bookkeeping. Must be
1005 // done before resizing.
1006 post_compact();
1007
1008 size_policy->major_collection_end();
1009
1010 size_policy->sample_old_gen_used_bytes(MAX2(pre_gc_values.old_gen_used(), old_gen->used_in_bytes()));
1011
1012 if (UseAdaptiveSizePolicy) {
1153 for (uint i = 0; i < num_workers; ++i) {
1154 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(i);
1155 cm->flush_and_destroy_marking_stats_cache();
1156 }
1157 }
1158
1159 class PSParallelCleaningTask : public WorkerTask {
1160 bool _unloading_occurred;
1161 CodeCacheUnloadingTask _code_cache_task;
1162 // Prune dead klasses from subklass/sibling/implementor lists.
1163 KlassCleaningTask _klass_cleaning_task;
1164
1165 public:
1166 PSParallelCleaningTask(bool unloading_occurred) :
1167 WorkerTask("PS Parallel Cleaning"),
1168 _unloading_occurred(unloading_occurred),
1169 _code_cache_task(unloading_occurred),
1170 _klass_cleaning_task() {}
1171
1172 void work(uint worker_id) {
1173 // Do first pass of code cache cleaning.
1174 _code_cache_task.work(worker_id);
1175
1176 // Clean all klasses that were not unloaded.
1177 // The weak metadata in klass doesn't need to be
1178 // processed if there was no unloading.
1179 if (_unloading_occurred) {
1180 _klass_cleaning_task.work();
1181 }
1182 }
1183 };
1184
1185 void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
1186 // Recursively traverse all live objects and mark them
1187 GCTraceTime(Info, gc, phases) tm("Marking Phase", &_gc_timer);
1188
1189 uint active_gc_threads = ParallelScavengeHeap::heap()->workers().active_workers();
1190
1191 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_mark);
1192 {
1419
1420 // Split [start, end) evenly for a number of workers and return the
1421 // range for worker_id.
1422 static void split_regions_for_worker(size_t start, size_t end,
1423 uint worker_id, uint num_workers,
1424 size_t* worker_start, size_t* worker_end) {
1425 assert(start < end, "precondition");
1426 assert(num_workers > 0, "precondition");
1427 assert(worker_id < num_workers, "precondition");
1428
1429 size_t num_regions = end - start;
1430 size_t num_regions_per_worker = num_regions / num_workers;
1431 size_t remainder = num_regions % num_workers;
1432 // The first few workers will get one extra.
1433 *worker_start = start + worker_id * num_regions_per_worker
1434 + MIN2(checked_cast<size_t>(worker_id), remainder);
1435 *worker_end = *worker_start + num_regions_per_worker
1436 + (worker_id < remainder ? 1 : 0);
1437 }
1438
1439 static bool safe_to_read_header(size_t words) {
1440 precond(words > 0);
1441
1442 // Safe to read if we have enough words for the full header, i.e., both
1443 // markWord and Klass pointer.
1444 const bool safe = words >= (size_t)oopDesc::header_size();
1445
1446 // If using Compact Object Headers, the full header is inside the markWord,
1447 // so will always be safe to read
1448 assert(!UseCompactObjectHeaders || safe, "Compact Object Headers should always be safe");
1449
1450 return safe;
1451 }
1452
1453 void PSParallelCompact::forward_to_new_addr() {
1454 GCTraceTime(Info, gc, phases) tm("Forward", &_gc_timer);
1455 uint nworkers = ParallelScavengeHeap::heap()->workers().active_workers();
1456
1457 struct ForwardTask final : public WorkerTask {
1458 uint _num_workers;
1459
1460 explicit ForwardTask(uint num_workers) :
1461 WorkerTask("PSForward task"),
1462 _num_workers(num_workers) {}
1463
1464 static bool should_preserve_mark(oop obj, HeapWord* end_addr) {
1465 size_t remaining_words = pointer_delta(end_addr, cast_from_oop<HeapWord*>(obj));
1466
1467 if (Arguments::is_valhalla_enabled() && !safe_to_read_header(remaining_words)) {
1468 // When using Valhalla, it might be necessary to preserve the Valhalla-
1469 // specific bits in the markWord. If the entire object header is
1470 // copied, the correct markWord (with the appropriate Valhalla bits)
1471 // can be safely read from the Klass. However, if the full header is
1472 // not copied, we cannot safely read the Klass to obtain this information.
1473 // In such cases, we always preserve the markWord to ensure that all
1474 // relevant bits, including Valhalla-specific ones, are retained.
1475 return true;
1476 } else {
1477 return obj->mark().must_be_preserved();
1478 }
1479 }
1480
1481 static void forward_objs_in_range(ParCompactionManager* cm,
1482 HeapWord* start,
1483 HeapWord* end,
1484 HeapWord* destination) {
1485 HeapWord* cur_addr = start;
1486 HeapWord* new_addr = destination;
1487
1488 while (cur_addr < end) {
1489 cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end);
1490 if (cur_addr >= end) {
1491 return;
1492 }
1493 assert(mark_bitmap()->is_marked(cur_addr), "inv");
1494 oop obj = cast_to_oop(cur_addr);
1495
1496 if (new_addr != cur_addr) {
1497 if (should_preserve_mark(obj, end)) {
1498 cm->preserved_marks()->push_always(obj, obj->mark());
1499 }
1500
1501 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
1502 }
1503 size_t obj_size = obj->size();
1504 new_addr += obj_size;
1505 cur_addr += obj_size;
1506 }
1507 }
1508
1509 void work(uint worker_id) override {
1510 ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
1511 for (uint id = old_space_id; id < last_space_id; ++id) {
1512 MutableSpace* sp = PSParallelCompact::space(SpaceId(id));
1513 HeapWord* dense_prefix_addr = dense_prefix(SpaceId(id));
1514 HeapWord* top = sp->top();
1515
1516 if (dense_prefix_addr == top) {
1517 // Empty space
1518 continue;
1519 }
1520
2121
2122 HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) {
2123 ParallelCompactData& sd = summary_data();
2124 assert(sd.is_region_aligned(region_start_addr), "precondition");
2125
2126 // Use per-region partial_obj_size to locate the end of the obj, that extends
2127 // to region_start_addr.
2128 size_t start_region_idx = sd.addr_to_region_idx(region_start_addr);
2129 size_t end_region_idx = sd.region_count();
2130 size_t accumulated_size = 0;
2131 for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) {
2132 size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size();
2133 accumulated_size += cur_partial_obj_size;
2134 if (cur_partial_obj_size != ParallelCompactData::RegionSize) {
2135 break;
2136 }
2137 }
2138 return region_start_addr + accumulated_size;
2139 }
2140
2141 static markWord safe_mark_word_prototype(HeapWord* cur_addr, HeapWord* end_addr) {
2142 // If the original markWord contains bits that cannot be reconstructed because
2143 // the header cannot be safely read, a placeholder is used. In this case,
2144 // the correct markWord is preserved before compaction and restored after
2145 // compaction completes.
2146 size_t remaining_words = pointer_delta(end_addr, cur_addr);
2147
2148 if (UseCompactObjectHeaders || (Arguments::is_valhalla_enabled() && safe_to_read_header(remaining_words))) {
2149 return cast_to_oop(cur_addr)->klass()->prototype_header();
2150 } else {
2151 return markWord::prototype();
2152 }
2153 }
2154
2155 // Use region_idx as the destination region, and evacuate all live objs on its
2156 // source regions to this destination region.
2157 void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx)
2158 {
2159 ParMarkBitMap* const bitmap = mark_bitmap();
2160 ParallelCompactData& sd = summary_data();
2161 RegionData* const region_ptr = sd.region(region_idx);
2162
2163 // Get the source region and related info.
2164 size_t src_region_idx = region_ptr->source_region();
2165 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx));
2166 HeapWord* src_space_top = _space_info[src_space_id].space()->top();
2167 HeapWord* dest_addr = sd.region_to_addr(region_idx);
2168
2169 closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx));
2170
2171 // Adjust src_region_idx to prepare for decrementing destination counts (the
2172 // destination count is not decremented when a region is copied to itself).
2173 if (src_region_idx == region_idx) {
2174 src_region_idx += 1;
2250 HeapWord* cur_addr = closure.source();
2251 HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1),
2252 src_space_top);
2253 // To handle the case where the final obj in source region extends to next region.
2254 HeapWord* final_obj_start = (end_addr == src_space_top)
2255 ? nullptr
2256 : sd.addr_to_region_ptr(end_addr)->partial_obj_addr();
2257 // Apply closure on objs inside [cur_addr, end_addr)
2258 do {
2259 cur_addr = bitmap->find_obj_beg(cur_addr, end_addr);
2260 if (cur_addr == end_addr) {
2261 break;
2262 }
2263 size_t obj_size;
2264 if (final_obj_start == cur_addr) {
2265 obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr);
2266 } else {
2267 // This obj doesn't extend into next region; size() is safe to use.
2268 obj_size = cast_to_oop(cur_addr)->size();
2269 }
2270
2271 markWord mark = safe_mark_word_prototype(cur_addr, end_addr);
2272
2273 // Perform the move and update of the object
2274 closure.do_addr(cur_addr, obj_size, mark);
2275
2276 cur_addr += obj_size;
2277 } while (cur_addr < end_addr && !closure.is_full());
2278
2279 if (closure.is_full()) {
2280 decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source());
2281 closure.complete_region(dest_addr, region_ptr);
2282 return;
2283 }
2284
2285 decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr);
2286
2287 // Move to the next source region, possibly switching spaces as well. All
2288 // args except end_addr may be modified.
2289 src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr);
2290 } while (true);
2291 }
2292
2293 void PSParallelCompact::fill_and_update_region(ParCompactionManager* cm, size_t region_idx)
2294 {
2295 MoveAndUpdateClosure cl(mark_bitmap(), region_idx);
2375 }
2376
2377 void MoveAndUpdateClosure::copy_partial_obj(size_t partial_obj_size)
2378 {
2379 size_t words = MIN2(partial_obj_size, words_remaining());
2380
2381 // This test is necessary; if omitted, the pointer updates to a partial object
2382 // that crosses the dense prefix boundary could be overwritten.
2383 if (source() != copy_destination()) {
2384 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2385 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2386 }
2387 update_state(words);
2388 }
2389
2390 void MoveAndUpdateClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2391 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::NormalRegion, "Region should be finished");
2392 region_ptr->set_completed();
2393 }
2394
2395 void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words, markWord mark) {
2396 assert(destination() != nullptr, "sanity");
2397 _source = addr;
2398
2399 // The start_array must be updated even if the object is not moving.
2400 if (_start_array != nullptr) {
2401 _start_array->update_for_block(destination(), destination() + words);
2402 }
2403
2404 // Avoid overflow
2405 words = MIN2(words, words_remaining());
2406 assert(words > 0, "inv");
2407
2408 if (copy_destination() != source()) {
2409 DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
2410 assert(source() != destination(), "inv");
2411 assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
2412 assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
2413 Copy::aligned_conjoint_words(source(), copy_destination(), words);
2414 cast_to_oop(copy_destination())->set_mark(mark);
2415 }
2416
2417 update_state(words);
2418 }
2419
2420 void MoveAndUpdateShadowClosure::complete_region(HeapWord* dest_addr, PSParallelCompact::RegionData* region_ptr) {
2421 assert(region_ptr->shadow_state() == ParallelCompactData::RegionData::ShadowRegion, "Region should be shadow");
2422 // Record the shadow region index
2423 region_ptr->set_shadow_region(_shadow);
2424 // Mark the shadow region as filled to indicate the data is ready to be
2425 // copied back
2426 region_ptr->mark_filled();
2427 // Try to copy the content of the shadow region back to its corresponding
2428 // heap region if available; the GC thread that decreases the destination
2429 // count to zero will do the copying otherwise (see
2430 // PSParallelCompact::decrement_destination_counts).
2431 if (((region_ptr->available() && region_ptr->claim()) || region_ptr->claimed()) && region_ptr->mark_copied()) {
2432 region_ptr->set_completed();
2433 PSParallelCompact::copy_back(PSParallelCompact::summary_data().region_to_addr(_shadow), dest_addr);
2434 ParCompactionManager::push_shadow_region_mt_safe(_shadow);
|