1 /*
2 * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/classLoaderDataGraph.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "code/codeCache.hpp"
30 #include "compiler/oopMap.hpp"
31 #include "gc/parallel/objectStartArray.inline.hpp"
32 #include "gc/parallel/parallelArguments.hpp"
33 #include "gc/parallel/parallelScavengeHeap.inline.hpp"
34 #include "gc/parallel/parMarkBitMap.inline.hpp"
35 #include "gc/parallel/psAdaptiveSizePolicy.hpp"
36 #include "gc/parallel/psCompactionManagerNew.inline.hpp"
37 #include "gc/parallel/psOldGen.hpp"
38 #include "gc/parallel/psParallelCompactNew.inline.hpp"
39 #include "gc/parallel/psPromotionManager.inline.hpp"
40 #include "gc/parallel/psScavenge.hpp"
41 #include "gc/parallel/psYoungGen.hpp"
42 #include "gc/shared/classUnloadingContext.hpp"
43 #include "gc/shared/fullGCForwarding.inline.hpp"
44 #include "gc/shared/gcCause.hpp"
45 #include "gc/shared/gcHeapSummary.hpp"
46 #include "gc/shared/gcId.hpp"
47 #include "gc/shared/gcLocker.hpp"
48 #include "gc/shared/gcTimer.hpp"
49 #include "gc/shared/gcTrace.hpp"
50 #include "gc/shared/gcTraceTime.inline.hpp"
51 #include "gc/shared/gcVMOperations.hpp"
52 #include "gc/shared/isGCActiveMark.hpp"
53 #include "gc/shared/oopStorageSetParState.inline.hpp"
54 #include "gc/shared/preservedMarks.inline.hpp"
55 #include "gc/shared/referencePolicy.hpp"
56 #include "gc/shared/referenceProcessor.hpp"
57 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
58 #include "gc/shared/spaceDecorator.hpp"
59 #include "gc/shared/taskTerminator.hpp"
60 #include "gc/shared/weakProcessor.inline.hpp"
61 #include "gc/shared/workerPolicy.hpp"
62 #include "gc/shared/workerThread.hpp"
63 #include "gc/shared/workerUtils.hpp"
64 #include "logging/log.hpp"
65 #include "memory/iterator.inline.hpp"
66 #include "memory/memoryReserver.hpp"
67 #include "memory/metaspaceUtils.hpp"
68 #include "memory/resourceArea.hpp"
69 #include "memory/universe.hpp"
70 #include "nmt/memTracker.hpp"
71 #include "oops/methodData.hpp"
72 #include "runtime/java.hpp"
73 #include "runtime/safepoint.hpp"
74 #include "runtime/threads.hpp"
75 #include "runtime/vmThread.hpp"
76 #include "services/memoryService.hpp"
77 #include "utilities/align.hpp"
78 #include "utilities/debug.hpp"
79 #include "utilities/events.hpp"
80 #include "utilities/formatBuffer.hpp"
81 #include "utilities/macros.hpp"
82
83 SpaceInfoNew PSParallelCompactNew::_space_info[PSParallelCompactNew::last_space_id];
84
85 size_t PSParallelCompactNew::_num_regions;
86 PCRegionData* PSParallelCompactNew::_region_data_array;
87 size_t PSParallelCompactNew::_num_regions_serial;
88 PCRegionData* PSParallelCompactNew::_region_data_array_serial;
89 PCRegionData** PSParallelCompactNew::_per_worker_region_data;
90 bool PSParallelCompactNew::_serial = false;
91
92 SpanSubjectToDiscoveryClosure PSParallelCompactNew::_span_based_discoverer;
93 ReferenceProcessor* PSParallelCompactNew::_ref_processor = nullptr;
94
95 void PSParallelCompactNew::print_on(outputStream* st) {
96 _mark_bitmap.print_on(st);
97 }
98
99 STWGCTimer PSParallelCompactNew::_gc_timer;
100 ParallelOldTracer PSParallelCompactNew::_gc_tracer;
101 elapsedTimer PSParallelCompactNew::_accumulated_time;
102 unsigned int PSParallelCompactNew::_maximum_compaction_gc_num = 0;
103 CollectorCounters* PSParallelCompactNew::_counters = nullptr;
104 ParMarkBitMap PSParallelCompactNew::_mark_bitmap;
105
106 PSParallelCompactNew::IsAliveClosure PSParallelCompactNew::_is_alive_closure;
107
108 class PCAdjustPointerClosure: public BasicOopIterateClosure {
109 template <typename T>
110 void do_oop_work(T* p) { PSParallelCompactNew::adjust_pointer(p); }
111
112 public:
113 void do_oop(oop* p) final { do_oop_work(p); }
114 void do_oop(narrowOop* p) final { do_oop_work(p); }
115
116 ReferenceIterationMode reference_iteration_mode() final { return DO_FIELDS; }
117 };
118
119 static PCAdjustPointerClosure pc_adjust_pointer_closure;
120
121 class IsAliveClosure: public BoolObjectClosure {
122 public:
123 bool do_object_b(oop p) final;
124 };
125
126
127 bool PSParallelCompactNew::IsAliveClosure::do_object_b(oop p) { return mark_bitmap()->is_marked(p); }
128
129 void PSParallelCompactNew::post_initialize() {
130 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
131 _span_based_discoverer.set_span(heap->reserved_region());
132 _ref_processor =
133 new ReferenceProcessor(&_span_based_discoverer,
134 ParallelGCThreads, // mt processing degree
135 ParallelGCThreads, // mt discovery degree
136 false, // concurrent_discovery
137 &_is_alive_closure); // non-header is alive closure
138
139 _counters = new CollectorCounters("Parallel full collection pauses", 1);
140
141 // Initialize static fields in ParCompactionManager.
142 ParCompactionManagerNew::initialize(mark_bitmap());
143 }
144
145 bool PSParallelCompactNew::initialize_aux_data() {
146 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
147 MemRegion mr = heap->reserved_region();
148 assert(mr.byte_size() != 0, "heap should be reserved");
149
150 initialize_space_info();
151
152 if (!_mark_bitmap.initialize(mr)) {
153 vm_shutdown_during_initialization(
154 err_msg("Unable to allocate %zuKB bitmaps for parallel "
155 "garbage collection for the requested %zuKB heap.",
156 _mark_bitmap.reserved_byte_size()/K, mr.byte_size()/K));
157 return false;
158 }
159
160 return true;
161 }
162
163 void PSParallelCompactNew::initialize_space_info()
164 {
165 memset(&_space_info, 0, sizeof(_space_info));
166
167 PSYoungGen* young_gen = ParallelScavengeHeap::heap()->young_gen();
168
169 _space_info[old_space_id].set_space(ParallelScavengeHeap::heap()->old_gen()->object_space());
170 _space_info[eden_space_id].set_space(young_gen->eden_space());
171 _space_info[from_space_id].set_space(young_gen->from_space());
172 _space_info[to_space_id].set_space(young_gen->to_space());
173
174 _space_info[old_space_id].set_start_array(ParallelScavengeHeap::heap()->old_gen()->start_array());
175 }
176
177 void
178 PSParallelCompactNew::clear_data_covering_space(SpaceId id)
179 {
180 // At this point, top is the value before GC, new_top() is the value that will
181 // be set at the end of GC. The marking bitmap is cleared to top; nothing
182 // should be marked above top.
183 MutableSpace* const space = _space_info[id].space();
184 HeapWord* const bot = space->bottom();
185 HeapWord* const top = space->top();
186
187 _mark_bitmap.clear_range(bot, top);
188 }
189
190 void PSParallelCompactNew::pre_compact()
191 {
192 // Update the from & to space pointers in space_info, since they are swapped
193 // at each young gen gc. Do the update unconditionally (even though a
194 // promotion failure does not swap spaces) because an unknown number of young
195 // collections will have swapped the spaces an unknown number of times.
196 GCTraceTime(Debug, gc, phases) tm("Pre Compact", &_gc_timer);
197 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
198 _space_info[from_space_id].set_space(ParallelScavengeHeap::heap()->young_gen()->from_space());
199 _space_info[to_space_id].set_space(ParallelScavengeHeap::heap()->young_gen()->to_space());
200
201 // Increment the invocation count
202 heap->increment_total_collections(true);
203
204 CodeCache::on_gc_marking_cycle_start();
205
206 heap->print_before_gc();
207 heap->trace_heap_before_gc(&_gc_tracer);
208
209 // Fill in TLABs
210 heap->ensure_parsability(true); // retire TLABs
211
212 if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) {
213 Universe::verify("Before GC");
214 }
215
216 DEBUG_ONLY(mark_bitmap()->verify_clear();)
217 }
218
219 void PSParallelCompactNew::post_compact()
220 {
221 GCTraceTime(Info, gc, phases) tm("Post Compact", &_gc_timer);
222
223 CodeCache::on_gc_marking_cycle_finish();
224 CodeCache::arm_all_nmethods();
225
226 for (unsigned int id = old_space_id; id < last_space_id; ++id) {
227 // Clear the marking bitmap, summary data and split info.
228 clear_data_covering_space(SpaceId(id));
229 }
230
231 {
232 PCRegionData* last_live[last_space_id];
233 for (uint i = old_space_id; i < last_space_id; ++i) {
234 last_live[i] = nullptr;
235 }
236
237 // Figure out last region in each space that has live data.
238 uint space_id = old_space_id;
239 MutableSpace* space = _space_info[space_id].space();
240 size_t num_regions = get_num_regions();
241 PCRegionData* region_data_array = get_region_data_array();
242 last_live[space_id] = ®ion_data_array[0];
243 for (size_t idx = 0; idx < num_regions; idx++) {
244 PCRegionData* rd = region_data_array + idx;
245 if(!space->contains(rd->bottom())) {
246 ++space_id;
247 assert(space_id < last_space_id, "invariant");
248 space = _space_info[space_id].space();
249 log_develop_trace(gc, compaction)("Last live for space: %u: %zu", space_id, idx);
250 last_live[space_id] = rd;
251 }
252 assert(space->contains(rd->bottom()), "next space should contain next region");
253 log_develop_trace(gc, compaction)("post-compact region: idx: %zu, bottom: " PTR_FORMAT ", new_top: " PTR_FORMAT ", end: " PTR_FORMAT, rd->idx(), p2i(rd->bottom()), p2i(rd->new_top()), p2i(rd->end()));
254 if (rd->new_top() > rd->bottom()) {
255 last_live[space_id] = rd;
256 log_develop_trace(gc, compaction)("Bump last live for space: %u", space_id);
257 }
258 }
259
260 for (uint i = old_space_id; i < last_space_id; ++i) {
261 PCRegionData* rd = last_live[i];
262 log_develop_trace(gc, compaction)(
263 "Last live region in space: %u, compaction region, " PTR_FORMAT ", #%zu: [" PTR_FORMAT ", " PTR_FORMAT "), new_top: " PTR_FORMAT,
264 i, p2i(rd), rd->idx(),
265 p2i(rd->bottom()), p2i(rd->end()), p2i(rd->new_top()));
266 }
267
268 // Fill all gaps and update the space boundaries.
269 space_id = old_space_id;
270 space = _space_info[space_id].space();
271 size_t total_live = 0;
272 size_t total_waste = 0;
273 for (size_t idx = 0; idx < num_regions; idx++) {
274 PCRegionData* rd = ®ion_data_array[idx];
275 PCRegionData* last_live_in_space = last_live[space_id];
276 assert(last_live_in_space != nullptr, "last live must not be null");
277 if (rd != last_live_in_space) {
278 if (rd->new_top() < rd->end()) {
279 ObjectStartArray* sa = start_array(SpaceId(space_id));
280 if (sa != nullptr) {
281 sa->update_for_block(rd->new_top(), rd->end());
282 }
283 ParallelScavengeHeap::heap()->fill_with_dummy_object(rd->new_top(), rd->end(), false);
284 }
285 size_t live = pointer_delta(rd->new_top(), rd->bottom());
286 size_t waste = pointer_delta(rd->end(), rd->new_top());
287 total_live += live;
288 total_waste += waste;
289 log_develop_trace(gc, compaction)(
290 "Live compaction region, #%zu: [" PTR_FORMAT ", " PTR_FORMAT "), new_top: " PTR_FORMAT ", live: %zu, waste: %zu",
291 rd->idx(),
292 p2i(rd->bottom()), p2i(rd->end()), p2i(rd->new_top()), live, waste);
293 } else {
294 // Update top of space.
295 space->set_top(rd->new_top());
296 size_t live = pointer_delta(rd->new_top(), rd->bottom());
297 total_live += live;
298 log_develop_trace(gc, compaction)(
299 "Live compaction region, #%zu: [" PTR_FORMAT ", " PTR_FORMAT "), new_top: " PTR_FORMAT ", live: %zu, waste: %zu",
300 rd->idx(),
301 p2i(rd->bottom()), p2i(rd->end()), p2i(rd->new_top()), live, size_t(0));
302
303 // Fast-Forward to next space.
304 for (; idx < num_regions - 1; idx++) {
305 rd = ®ion_data_array[idx + 1];
306 if (!space->contains(rd->bottom())) {
307 space_id++;
308 assert(space_id < last_space_id, "must be");
309 space = _space_info[space_id].space();
310 assert(space->contains(rd->bottom()), "space must contain region");
311 break;
312 }
313 }
314 }
315 }
316 log_debug(gc, compaction)("total live: %zu, total waste: %zu, ratio: %f", total_live, total_waste, ((float)total_waste)/((float)(total_live + total_waste)));
317 }
318 {
319 FREE_C_HEAP_ARRAY(_per_worker_region_data);
320 FREE_C_HEAP_ARRAY(_region_data_array);
321 FREE_C_HEAP_ARRAY(_region_data_array_serial);
322 }
323 #ifdef ASSERT
324 {
325 mark_bitmap()->verify_clear();
326 }
327 #endif
328
329 ParCompactionManagerNew::flush_all_string_dedup_requests();
330
331 MutableSpace* const eden_space = _space_info[eden_space_id].space();
332 MutableSpace* const from_space = _space_info[from_space_id].space();
333 MutableSpace* const to_space = _space_info[to_space_id].space();
334
335 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
336 bool eden_empty = eden_space->is_empty();
337
338 // Update heap occupancy information which is used as input to the soft ref
339 // clearing policy at the next gc.
340 Universe::heap()->update_capacity_and_used_at_gc();
341
342 bool young_gen_empty = eden_empty && from_space->is_empty() &&
343 to_space->is_empty();
344
345 PSCardTable* ct = heap->card_table();
346 MemRegion old_mr = ParallelScavengeHeap::heap()->old_gen()->committed();
347 if (young_gen_empty) {
348 ct->clear_MemRegion(old_mr);
349 } else {
350 ct->dirty_MemRegion(old_mr);
351 }
352
353 {
354 // Delete metaspaces for unloaded class loaders and clean up loader_data graph
355 GCTraceTime(Debug, gc, phases) t("Purge Class Loader Data", gc_timer());
356 ClassLoaderDataGraph::purge(true /* at_safepoint */);
357 DEBUG_ONLY(MetaspaceUtils::verify();)
358 }
359
360 // Need to clear claim bits for the next mark.
361 ClassLoaderDataGraph::clear_claimed_marks();
362
363 heap->prune_scavengable_nmethods();
364
365 #ifdef COMPILER2
366 DerivedPointerTable::update_pointers();
367 #endif
368
369 // Signal that we have completed a visit to all live objects.
370 Universe::heap()->record_whole_heap_examined_timestamp();
371 }
372
373 void PSParallelCompactNew::setup_regions_parallel() {
374 const size_t region_size_words = calculate_region_size();
375
376 size_t num_regions = 0;
377 for (uint i = old_space_id; i < last_space_id; ++i) {
378 MutableSpace* const space = _space_info[i].space();
379 size_t const space_size_words = space->capacity_in_words();
380 num_regions += align_up(space_size_words, region_size_words) / region_size_words;
381 }
382 _region_data_array = NEW_C_HEAP_ARRAY(PCRegionData, num_regions, mtGC);
383
384 size_t region_idx = 0;
385 for (uint i = old_space_id; i < last_space_id; ++i) {
386 const MutableSpace* space = _space_info[i].space();
387 HeapWord* addr = space->bottom();
388 HeapWord* sp_end = space->end();
389 HeapWord* sp_top = space->top();
390 while (addr < sp_end) {
391 HeapWord* end = MIN2(align_up(addr + region_size_words, region_size_words), space->end());
392 if (addr < sp_top) {
393 HeapWord* prev_obj_start = _mark_bitmap.find_obj_beg_reverse(addr, end);
394 if (prev_obj_start < end) {
395 HeapWord* prev_obj_end = prev_obj_start + cast_to_oop(prev_obj_start)->size();
396 if (end < prev_obj_end) {
397 // Object crosses region boundary, adjust end to be after object's last word.
398 end = prev_obj_end;
399 }
400 }
401 }
402 assert(region_idx < num_regions, "must not exceed number of regions: region_idx: %zu, num_regions: %zu", region_idx, num_regions);
403 HeapWord* top;
404 if (sp_top < addr) {
405 top = addr;
406 } else if (sp_top >= end) {
407 top = end;
408 } else {
409 top = sp_top;
410 }
411 assert(ParallelScavengeHeap::heap()->is_in_reserved(addr), "addr must be in heap: " PTR_FORMAT, p2i(addr));
412 new (_region_data_array + region_idx) PCRegionData(region_idx, addr, top, end);
413 addr = end;
414 region_idx++;
415 }
416 }
417 _num_regions = region_idx;
418 log_info(gc)("Number of regions: %zu, region size: " EXACTFMT, _num_regions, EXACTFMTARGS(region_size_words * HeapWordSize));
419 }
420
421 size_t PSParallelCompactNew::calculate_region_size() {
422 // Minimum 0.5MB region size
423 static const size_t FLOOR_REGION_SIZE_WORDS = (SpaceAlignment / HeapWordSize);
424 size_t total_heap_words = 0;
425 for (uint i = old_space_id; i < last_space_id; ++i) {
426 total_heap_words += _space_info[i].space()->capacity_in_words();
427 }
428
429 // Per-worker region count for dynamic region sizing
430 static const uint REGIONS_PER_WORKER = 20; // Based on 5% boundary waste threshold
431 const uint max_workers = ParallelScavengeHeap::heap()->workers().max_workers();
432 const size_t total_regions_count = (size_t)max_workers * REGIONS_PER_WORKER;
433 const size_t dynamic_region_size_words = total_heap_words / total_regions_count;
434 return round_up_power_of_2(MAX2(dynamic_region_size_words, FLOOR_REGION_SIZE_WORDS));
435 }
436
437 void PSParallelCompactNew::setup_regions_serial() {
438 _num_regions_serial = last_space_id;
439 _region_data_array_serial = NEW_C_HEAP_ARRAY(PCRegionData, _num_regions_serial, mtGC);
440 new (_region_data_array_serial + old_space_id) PCRegionData(old_space_id, space(old_space_id)->bottom(), space(old_space_id)->top(), space(old_space_id)->end());
441 new (_region_data_array_serial + eden_space_id) PCRegionData(eden_space_id, space(eden_space_id)->bottom(), space(eden_space_id)->top(), space(eden_space_id)->end());
442 new (_region_data_array_serial + from_space_id) PCRegionData(from_space_id, space(from_space_id)->bottom(), space(from_space_id)->top(), space(from_space_id)->end());
443 new (_region_data_array_serial + to_space_id) PCRegionData(to_space_id, space(to_space_id)->bottom(), space(to_space_id)->top(), space(to_space_id)->end());
444 }
445
446 bool PSParallelCompactNew::check_maximum_compaction(bool should_do_max_compaction) {
447
448 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
449
450 // Check System.GC
451 bool is_max_on_system_gc = UseMaximumCompactionOnSystemGC
452 && GCCause::is_user_requested_gc(heap->gc_cause());
453
454 return should_do_max_compaction
455 || is_max_on_system_gc;
456 }
457
458 void PSParallelCompactNew::summary_phase() {
459 GCTraceTime(Info, gc, phases) tm("Summary Phase", &_gc_timer);
460
461 setup_regions_serial();
462 setup_regions_parallel();
463
464 #ifndef PRODUCT
465 for (size_t idx = 0; idx < _num_regions; idx++) {
466 PCRegionData* rd = &_region_data_array[idx];
467 log_develop_trace(gc, compaction)("Compaction region #%zu: [" PTR_FORMAT ", " PTR_FORMAT ")", rd->idx(), p2i(
468 rd->bottom()), p2i(rd->end()));
469 }
470 #endif
471 }
472
473 // This method should contain all heap-specific policy for invoking a full
474 // collection. invoke_no_policy() will only attempt to compact the heap; it
475 // will do nothing further. If we need to bail out for policy reasons, scavenge
476 // before full gc, or any other specialized behavior, it needs to be added here.
477 //
478 // Note that this method should only be called from the vm_thread while at a
479 // safepoint.
480 //
481 // Note that the all_soft_refs_clear flag in the soft ref policy
482 // may be true because this method can be called without intervening
483 // activity. For example when the heap space is tight and full measure
484 // are being taken to free space.
485 bool PSParallelCompactNew::invoke(bool clear_all_soft_refs, bool should_do_max_compaction) {
486 assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
487 assert(Thread::current() == (Thread*)VMThread::vm_thread(),
488 "should be in vm thread");
489
490 SvcGCMarker sgcm(SvcGCMarker::FULL);
491 IsSTWGCActiveMark mark;
492
493 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
494
495 GCIdMark gc_id_mark;
496 _gc_timer.register_gc_start();
497 _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start());
498
499 GCCause::Cause gc_cause = heap->gc_cause();
500 PSYoungGen* young_gen = ParallelScavengeHeap::heap()->young_gen();
501 PSOldGen* old_gen = ParallelScavengeHeap::heap()->old_gen();
502 PSAdaptiveSizePolicy* size_policy = heap->size_policy();
503
504 // Make sure data structures are sane, make the heap parsable, and do other
505 // miscellaneous bookkeeping.
506 pre_compact();
507
508 const PreGenGCValues pre_gc_values = heap->get_pre_gc_values();
509
510 {
511 const uint active_workers =
512 WorkerPolicy::calc_active_workers(ParallelScavengeHeap::heap()->workers().max_workers(),
513 ParallelScavengeHeap::heap()->workers().active_workers(),
514 Threads::number_of_non_daemon_threads());
515 ParallelScavengeHeap::heap()->workers().set_active_workers(active_workers);
516
517 if (check_maximum_compaction(should_do_max_compaction)) {
518 // Serial compaction executes the forwarding and compaction phases serially,
519 // thus achieving perfect compaction.
520 // Marking and ajust-references would still be executed in parallel threads.
521 _serial = true;
522 } else {
523 _serial = false;
524 }
525
526 GCTraceCPUTime tcpu(&_gc_tracer);
527 GCTraceTime(Info, gc) tm("Pause Full", nullptr, gc_cause, true);
528
529 heap->pre_full_gc_dump(&_gc_timer);
530
531 TraceCollectorStats tcs(counters());
532 TraceMemoryManagerStats tms(heap->old_gc_manager(), gc_cause, "end of major GC");
533
534 if (log_is_enabled(Debug, gc, heap, exit)) {
535 accumulated_time()->start();
536 }
537
538 // Let the size policy know we're starting
539 size_policy->major_collection_begin();
540
541 #ifdef COMPILER2
542 DerivedPointerTable::clear();
543 #endif
544
545 ref_processor()->start_discovery(clear_all_soft_refs);
546
547 ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
548 false /* unregister_nmethods_during_purge */,
549 false /* lock_nmethod_free_separately */);
550
551 marking_phase(&_gc_tracer);
552
553 summary_phase();
554
555 #ifdef COMPILER2
556 assert(DerivedPointerTable::is_active(), "Sanity");
557 DerivedPointerTable::set_active(false);
558 #endif
559
560 FullGCForwarding::begin();
561
562 forward_to_new_addr();
563
564 adjust_pointers();
565
566 compact();
567
568 FullGCForwarding::end();
569
570 ParCompactionManagerNew::_preserved_marks_set->restore(&ParallelScavengeHeap::heap()->workers());
571
572 // Reset the mark bitmap, summary data, and do other bookkeeping. Must be
573 // done before resizing.
574 post_compact();
575
576 // Let the size policy know we're done
577 size_policy->major_collection_end();
578
579 size_policy->sample_old_gen_used_bytes(MAX2(pre_gc_values.old_gen_used(), old_gen->used_in_bytes()));
580
581 if (UseAdaptiveSizePolicy) {
582 heap->resize_after_full_gc();
583 }
584
585 heap->resize_all_tlabs();
586
587 // Resize the metaspace capacity after a collection
588 MetaspaceGC::compute_new_size();
589
590 if (log_is_enabled(Debug, gc, heap, exit)) {
591 accumulated_time()->stop();
592 }
593
594 heap->print_heap_change(pre_gc_values);
595
596 // Track memory usage and detect low memory
597 MemoryService::track_memory_usage();
598 heap->update_counters();
599
600 heap->post_full_gc_dump(&_gc_timer);
601
602 size_policy->record_gc_pause_end_instant();
603 }
604
605 heap->gc_epilogue(true);
606
607 if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) {
608 Universe::verify("After GC");
609 }
610
611 heap->print_after_gc();
612 heap->trace_heap_after_gc(&_gc_tracer);
613
614 _gc_timer.register_gc_end();
615
616 _gc_tracer.report_gc_end(_gc_timer.gc_end(), _gc_timer.time_partitions());
617
618 return true;
619 }
620
621 class PCAddThreadRootsMarkingTaskClosureNew : public ThreadClosure {
622 private:
623 uint _worker_id;
624
625 public:
626 explicit PCAddThreadRootsMarkingTaskClosureNew(uint worker_id) : _worker_id(worker_id) { }
627 void do_thread(Thread* thread) final {
628 assert(ParallelScavengeHeap::heap()->is_stw_gc_active(), "called outside gc");
629
630 ResourceMark rm;
631
632 ParCompactionManagerNew* cm = ParCompactionManagerNew::gc_thread_compaction_manager(_worker_id);
633
634 MarkingNMethodClosure mark_and_push_in_blobs(&cm->_mark_and_push_closure);
635
636 thread->oops_do(&cm->_mark_and_push_closure, &mark_and_push_in_blobs);
637
638 // Do the real work
639 cm->follow_marking_stacks();
640 }
641 };
642
643 void steal_marking_work_new(TaskTerminator& terminator, uint worker_id) {
644 assert(ParallelScavengeHeap::heap()->is_stw_gc_active(), "called outside gc");
645
646 ParCompactionManagerNew* cm =
647 ParCompactionManagerNew::gc_thread_compaction_manager(worker_id);
648
649 do {
650 ScannerTask task;
651 if (ParCompactionManagerNew::steal(worker_id, task)) {
652 cm->follow_contents(task, true);
653 }
654 cm->follow_marking_stacks();
655 } while (!terminator.offer_termination());
656 }
657
658 class MarkFromRootsTaskNew : public WorkerTask {
659 NMethodMarkingScope _nmethod_marking_scope;
660 ThreadsClaimTokenScope _threads_claim_token_scope;
661 OopStorageSetStrongParState<false /* concurrent */, false /* is_const */> _oop_storage_set_par_state;
662 TaskTerminator _terminator;
663 uint _active_workers;
664
665 public:
666 explicit MarkFromRootsTaskNew(uint active_workers) :
667 WorkerTask("MarkFromRootsTaskNew"),
668 _nmethod_marking_scope(),
669 _threads_claim_token_scope(),
670 _terminator(active_workers, ParCompactionManagerNew::marking_stacks()),
671 _active_workers(active_workers) {}
672
673 void work(uint worker_id) final {
674 ParCompactionManagerNew* cm = ParCompactionManagerNew::gc_thread_compaction_manager(worker_id);
675 {
676 CLDToOopClosure cld_closure(&cm->_mark_and_push_closure, ClassLoaderData::_claim_stw_fullgc_mark);
677 ClassLoaderDataGraph::always_strong_cld_do(&cld_closure);
678
679 // Do the real work
680 cm->follow_marking_stacks();
681 }
682
683 {
684 PCAddThreadRootsMarkingTaskClosureNew closure(worker_id);
685 Threads::possibly_parallel_threads_do(_active_workers > 1 /* is_par */, &closure);
686 }
687
688 // Mark from OopStorages
689 {
690 _oop_storage_set_par_state.oops_do(&cm->_mark_and_push_closure);
691 // Do the real work
692 cm->follow_marking_stacks();
693 }
694
695 if (_active_workers > 1) {
696 steal_marking_work_new(_terminator, worker_id);
697 }
698 }
699 };
700
701 class ParallelCompactRefProcProxyTaskNew : public RefProcProxyTask {
702 TaskTerminator _terminator;
703
704 public:
705 explicit ParallelCompactRefProcProxyTaskNew(uint max_workers)
706 : RefProcProxyTask("ParallelCompactRefProcProxyTaskNew", max_workers),
707 _terminator(_max_workers, ParCompactionManagerNew::marking_stacks()) {}
708
709 void work(uint worker_id) final {
710 assert(worker_id < _max_workers, "sanity");
711 ParCompactionManagerNew* cm = (_tm == RefProcThreadModel::Single) ? ParCompactionManagerNew::get_vmthread_cm() : ParCompactionManagerNew::gc_thread_compaction_manager(worker_id);
712 BarrierEnqueueDiscoveredFieldClosure enqueue;
713 ParCompactionManagerNew::FollowStackClosure complete_gc(cm, (_tm == RefProcThreadModel::Single) ? nullptr : &_terminator, worker_id);
714 _rp_task->rp_work(worker_id, PSParallelCompactNew::is_alive_closure(), &cm->_mark_and_push_closure, &enqueue, &complete_gc);
715 }
716
717 void prepare_run_task_hook() final {
718 _terminator.reset_for_reuse(_queue_count);
719 }
720 };
721
722 void PSParallelCompactNew::marking_phase(ParallelOldTracer *gc_tracer) {
723 // Recursively traverse all live objects and mark them
724 GCTraceTime(Info, gc, phases) tm("Marking Phase", &_gc_timer);
725
726 uint active_gc_threads = ParallelScavengeHeap::heap()->workers().active_workers();
727
728 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_mark);
729 {
730 GCTraceTime(Debug, gc, phases) pm_tm("Par Mark", &_gc_timer);
731
732 MarkFromRootsTaskNew task(active_gc_threads);
733 ParallelScavengeHeap::heap()->workers().run_task(&task);
734 }
735
736 // Process reference objects found during marking
737 {
738 GCTraceTime(Debug, gc, phases) rp_tm("Reference Processing", &_gc_timer);
739
740 ReferenceProcessorStats stats;
741 ReferenceProcessorPhaseTimes pt(&_gc_timer, ref_processor()->max_num_queues());
742
743 ParallelCompactRefProcProxyTaskNew task(ref_processor()->max_num_queues());
744 stats = ref_processor()->process_discovered_references(task, &ParallelScavengeHeap::heap()->workers(), pt);
745
746 gc_tracer->report_gc_reference_stats(stats);
747 pt.print_all_references();
748 }
749
750 // This is the point where the entire marking should have completed.
751 ParCompactionManagerNew::verify_all_marking_stack_empty();
752
753 {
754 GCTraceTime(Debug, gc, phases) wp_tm("Weak Processing", &_gc_timer);
755 WeakProcessor::weak_oops_do(&ParallelScavengeHeap::heap()->workers(),
756 is_alive_closure(),
757 &do_nothing_cl,
758 1);
759 }
760
761 {
762 GCTraceTime(Debug, gc, phases) tm_m("Class Unloading", &_gc_timer);
763
764 ClassUnloadingContext* ctx = ClassUnloadingContext::context();
765
766 bool unloading_occurred;
767 {
768 CodeCache::UnlinkingScope scope(is_alive_closure());
769
770 // Follow system dictionary roots and unload classes.
771 unloading_occurred = SystemDictionary::do_unloading(&_gc_timer);
772
773 // Unload nmethods.
774 CodeCache::do_unloading(unloading_occurred);
775 }
776
777 {
778 GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", gc_timer());
779 // Release unloaded nmethod's memory.
780 ctx->purge_nmethods();
781 }
782 {
783 GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", &_gc_timer);
784 ParallelScavengeHeap::heap()->prune_unlinked_nmethods();
785 }
786 {
787 GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
788 ctx->free_nmethods();
789 }
790
791 // Prune dead klasses from subklass/sibling/implementor lists.
792 Klass::clean_weak_klass_links(unloading_occurred);
793 }
794
795 {
796 GCTraceTime(Debug, gc, phases) roc_tm("Report Object Count", &_gc_timer);
797 _gc_tracer.report_object_count_after_gc(is_alive_closure(), &ParallelScavengeHeap::heap()->workers());
798 }
799 #if TASKQUEUE_STATS
800 ParCompactionManagerNew::print_and_reset_taskqueue_stats();
801 #endif
802 }
803
804 void PSParallelCompactNew::adjust_pointers_in_spaces(uint worker_id) {
805 auto start_time = Ticks::now();
806 for (size_t i = 0; i < _num_regions; i++) {
807 PCRegionData* region = &_region_data_array[i];
808 if (!region->claim()) {
809 continue;
810 }
811 log_trace(gc, compaction)("Adjusting pointers in region: %zu (worker_id: %u)", region->idx(), worker_id);
812 HeapWord* end = region->top();
813 HeapWord* current = _mark_bitmap.find_obj_beg(region->bottom(), end);
814 while (current < end) {
815 assert(_mark_bitmap.is_marked(current), "must be marked");
816 oop obj = cast_to_oop(current);
817 size_t size = obj->size();
818 obj->oop_iterate(&pc_adjust_pointer_closure);
819 current = _mark_bitmap.find_obj_beg(current + size, end);
820 }
821 }
822 log_trace(gc, phases)("adjust_pointers_in_spaces worker %u: %.3f ms", worker_id, (Ticks::now() - start_time).seconds() * 1000);
823 }
824
825 class PSAdjustTaskNew final : public WorkerTask {
826 SubTasksDone _sub_tasks;
827 WeakProcessor::Task _weak_proc_task;
828 OopStorageSetStrongParState<false, false> _oop_storage_iter;
829 uint _nworkers;
830
831 enum PSAdjustSubTask {
832 PSAdjustSubTask_code_cache,
833
834 PSAdjustSubTask_num_elements
835 };
836
837 public:
838 explicit PSAdjustTaskNew(uint nworkers) :
839 WorkerTask("PSAdjust task"),
840 _sub_tasks(PSAdjustSubTask_num_elements),
841 _weak_proc_task(nworkers),
842 _nworkers(nworkers) {
843
844 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
845 if (nworkers > 1) {
846 Threads::change_thread_claim_token();
847 }
848 }
849
850 ~PSAdjustTaskNew() {
851 Threads::assert_all_threads_claimed();
852 }
853
854 void work(uint worker_id) final {
855 ParCompactionManagerNew* cm = ParCompactionManagerNew::gc_thread_compaction_manager(worker_id);
856 cm->preserved_marks()->adjust_during_full_gc();
857 {
858 // adjust pointers in all spaces
859 PSParallelCompactNew::adjust_pointers_in_spaces(worker_id);
860 }
861 {
862 ResourceMark rm;
863 Threads::possibly_parallel_oops_do(_nworkers > 1, &pc_adjust_pointer_closure, nullptr);
864 }
865 _oop_storage_iter.oops_do(&pc_adjust_pointer_closure);
866 {
867 CLDToOopClosure cld_closure(&pc_adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
868 ClassLoaderDataGraph::cld_do(&cld_closure);
869 }
870 {
871 AlwaysTrueClosure always_alive;
872 _weak_proc_task.work(worker_id, &always_alive, &pc_adjust_pointer_closure);
873 }
874 if (_sub_tasks.try_claim_task(PSAdjustSubTask_code_cache)) {
875 NMethodToOopClosure adjust_code(&pc_adjust_pointer_closure, NMethodToOopClosure::FixRelocations);
876 CodeCache::nmethods_do(&adjust_code);
877 }
878 _sub_tasks.all_tasks_claimed();
879 }
880 };
881
882 void PSParallelCompactNew::adjust_pointers() {
883 // Adjust the pointers to reflect the new locations
884 GCTraceTime(Info, gc, phases) tm("Adjust Pointers", &_gc_timer);
885 uint num_workers = ParallelScavengeHeap::heap()->workers().active_workers();
886 PSAdjustTaskNew task(num_workers);
887 ParallelScavengeHeap::heap()->workers().run_task(&task);
888 }
889
890 void PSParallelCompactNew::forward_to_new_addr() {
891 GCTraceTime(Info, gc, phases) tm("Forward", &_gc_timer);
892 uint num_workers = get_num_workers();
893 _per_worker_region_data = NEW_C_HEAP_ARRAY(PCRegionData*, num_workers, mtGC);
894 for (uint i = 0; i < num_workers; i++) {
895 _per_worker_region_data[i] = nullptr;
896 }
897
898 class ForwardState {
899 uint _worker_id;
900 PCRegionData* _compaction_region;
901 HeapWord* _compaction_point;
902
903 void ensure_compaction_point() {
904 if (_compaction_point == nullptr) {
905 assert(_compaction_region == nullptr, "invariant");
906 _compaction_region = _per_worker_region_data[_worker_id];
907 assert(_compaction_region != nullptr, "invariant");
908 _compaction_point = _compaction_region->bottom();
909 }
910 }
911 public:
912 explicit ForwardState(uint worker_id) :
913 _worker_id(worker_id),
914 _compaction_region(nullptr),
915 _compaction_point(nullptr) {
916 }
917
918 size_t available() const {
919 return pointer_delta(_compaction_region->end(), _compaction_point);
920 }
921
922 void forward_objs_in_region(ParCompactionManagerNew* cm, PCRegionData* region) {
923 ensure_compaction_point();
924 HeapWord* end = region->end();
925 HeapWord* current = _mark_bitmap.find_obj_beg(region->bottom(), end);
926 while (current < end) {
927 assert(_mark_bitmap.is_marked(current), "must be marked");
928 oop obj = cast_to_oop(current);
929 assert(region->contains(obj), "object must not cross region boundary: obj: " PTR_FORMAT ", obj_end: " PTR_FORMAT ", region start: " PTR_FORMAT ", region end: " PTR_FORMAT, p2i(obj), p2i(cast_from_oop<HeapWord*>(obj) + obj->size()), p2i(region->bottom()), p2i(region->end()));
930 size_t old_size = obj->size();
931 size_t new_size = obj->copy_size(old_size, obj->mark());
932 size_t size = (current == _compaction_point) ? old_size : new_size;
933 while (size > available()) {
934 _compaction_region->set_new_top(_compaction_point);
935 _compaction_region = _compaction_region->local_next();
936 assert(_compaction_region != nullptr, "must find a compaction region");
937 _compaction_point = _compaction_region->bottom();
938 size = (current == _compaction_point) ? old_size : new_size;
939 }
940 //log_develop_trace(gc, compaction)("Forwarding obj: " PTR_FORMAT ", to: " PTR_FORMAT, p2i(obj), p2i(_compaction_point));
941 if (current != _compaction_point) {
942 cm->preserved_marks()->push_if_necessary(obj, obj->mark());
943 FullGCForwarding::forward_to(obj, cast_to_oop(_compaction_point));
944 }
945 _compaction_point += size;
946 assert(_compaction_point <= _compaction_region->end(), "object must fit in region");
947 current += old_size;
948 assert(current <= end, "object must not cross region boundary");
949 current = _mark_bitmap.find_obj_beg(current, end);
950 }
951 }
952 void finish() {
953 if (_compaction_region != nullptr) {
954 _compaction_region->set_new_top(_compaction_point);
955 }
956 }
957 };
958
959 struct ForwardTask final : public WorkerTask {
960 ForwardTask() : WorkerTask("PSForward task") {}
961
962 void work(uint worker_id) override {
963 ParCompactionManagerNew* cm = ParCompactionManagerNew::gc_thread_compaction_manager(worker_id);
964 ForwardState state(worker_id);
965 PCRegionData** last_link = &_per_worker_region_data[worker_id];
966 size_t idx = worker_id;
967 uint num_workers = get_num_workers();
968 size_t num_regions = get_num_regions();
969 PCRegionData* region_data_array = get_region_data_array();
970 while (idx < num_regions) {
971 PCRegionData* region = region_data_array + idx;
972 *last_link = region;
973 last_link = region->local_next_addr();
974 state.forward_objs_in_region(cm, region);
975 idx += num_workers;
976 }
977 state.finish();
978 }
979 } task;
980
981 uint par_workers = ParallelScavengeHeap::heap()->workers().active_workers();
982 ParallelScavengeHeap::heap()->workers().set_active_workers(num_workers);
983 ParallelScavengeHeap::heap()->workers().run_task(&task);
984 ParallelScavengeHeap::heap()->workers().set_active_workers(par_workers);
985
986 #ifndef PRODUCT
987 for (uint wid = 0; wid < num_workers; wid++) {
988 for (PCRegionData* rd = _per_worker_region_data[wid]; rd != nullptr; rd = rd->local_next()) {
989 log_develop_trace(gc, compaction)("Per worker compaction region, worker: %d, #%zu: [" PTR_FORMAT ", " PTR_FORMAT "), new_top: " PTR_FORMAT, wid, rd->idx(),
990 p2i(rd->bottom()), p2i(rd->end()), p2i(rd->new_top()));
991 }
992 }
993 #endif
994 }
995
996 void PSParallelCompactNew::compact() {
997 GCTraceTime(Info, gc, phases) tm("Compaction Phase", &_gc_timer);
998 class CompactTask final : public WorkerTask {
999 static void compact_region(PCRegionData* region) {
1000 HeapWord* bottom = region->bottom();
1001 HeapWord* end = region->top();
1002 if (bottom == end) {
1003 return;
1004 }
1005 HeapWord* current = _mark_bitmap.find_obj_beg(bottom, end);
1006 while (current < end) {
1007 oop obj = cast_to_oop(current);
1008 size_t size = obj->size();
1009 if (FullGCForwarding::is_forwarded(obj)) {
1010 oop fwd = FullGCForwarding::forwardee(obj);
1011 auto* dst = cast_from_oop<HeapWord*>(fwd);
1012 ObjectStartArray* sa = start_array(space_id(dst));
1013 if (sa != nullptr) {
1014 assert(dst != current, "expect moving object");
1015 size_t new_words = obj->copy_size(size, obj->mark());
1016 sa->update_for_block(dst, dst + new_words);
1017 }
1018
1019 Copy::aligned_conjoint_words(current, dst, size);
1020 fwd->reinit_mark();
1021 fwd->initialize_hash_if_necessary(obj);
1022 } else {
1023 // The start_array must be updated even if the object is not moving.
1024 ObjectStartArray* sa = start_array(space_id(current));
1025 if (sa != nullptr) {
1026 sa->update_for_block(current, current + size);
1027 }
1028 }
1029 current = _mark_bitmap.find_obj_beg(current + size, end);
1030 }
1031 }
1032 public:
1033 explicit CompactTask() : WorkerTask("PSCompact task") {}
1034 void work(uint worker_id) override {
1035 PCRegionData* region = _per_worker_region_data[worker_id];
1036 while (region != nullptr) {
1037 log_trace(gc)("Compact worker: %u, compacting region: %zu", worker_id, region->idx());
1038 compact_region(region);
1039 region = region->local_next();
1040 }
1041 }
1042 } task;
1043
1044 uint num_workers = get_num_workers();
1045 uint par_workers = ParallelScavengeHeap::heap()->workers().active_workers();
1046 ParallelScavengeHeap::heap()->workers().set_active_workers(num_workers);
1047 ParallelScavengeHeap::heap()->workers().run_task(&task);
1048 ParallelScavengeHeap::heap()->workers().set_active_workers(par_workers);
1049 }
1050
1051 // Return the SpaceId for the space containing addr. If addr is not in the
1052 // heap, last_space_id is returned. In debug mode it expects the address to be
1053 // in the heap and asserts such.
1054 PSParallelCompactNew::SpaceId PSParallelCompactNew::space_id(HeapWord* addr) {
1055 assert(ParallelScavengeHeap::heap()->is_in_reserved(addr), "addr not in the heap");
1056
1057 for (unsigned int id = old_space_id; id < last_space_id; ++id) {
1058 if (_space_info[id].space()->contains(addr)) {
1059 return SpaceId(id);
1060 }
1061 }
1062
1063 assert(false, "no space contains the addr");
1064 return last_space_id;
1065 }