1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/classLoaderData.inline.hpp"
26 #include "classfile/classLoaderDataGraph.hpp"
27 #include "classfile/javaClasses.inline.hpp"
28 #include "classfile/stringTable.hpp"
29 #include "classfile/symbolTable.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "classfile/vmSymbols.hpp"
32 #include "code/codeCache.hpp"
33 #include "compiler/compileBroker.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/serial/cardTableRS.hpp"
36 #include "gc/serial/defNewGeneration.hpp"
37 #include "gc/serial/serialFullGC.hpp"
38 #include "gc/serial/serialGcRefProcProxyTask.hpp"
39 #include "gc/serial/serialHeap.hpp"
40 #include "gc/serial/serialStringDedup.hpp"
41 #include "gc/serial/tenuredGeneration.inline.hpp"
42 #include "gc/shared/classUnloadingContext.hpp"
43 #include "gc/shared/collectedHeap.inline.hpp"
44 #include "gc/shared/continuationGCSupport.inline.hpp"
45 #include "gc/shared/fullGCForwarding.inline.hpp"
46 #include "gc/shared/gc_globals.hpp"
47 #include "gc/shared/gcHeapSummary.hpp"
48 #include "gc/shared/gcTimer.hpp"
49 #include "gc/shared/gcTrace.hpp"
50 #include "gc/shared/gcTraceTime.inline.hpp"
51 #include "gc/shared/oopStorageSet.inline.hpp"
52 #include "gc/shared/preservedMarks.inline.hpp"
53 #include "gc/shared/referencePolicy.hpp"
54 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
55 #include "gc/shared/space.hpp"
56 #include "gc/shared/weakProcessor.hpp"
57 #include "memory/iterator.inline.hpp"
58 #include "memory/universe.hpp"
59 #include "oops/access.inline.hpp"
60 #include "oops/compressedOops.inline.hpp"
61 #include "oops/instanceRefKlass.hpp"
62 #include "oops/markWord.hpp"
63 #include "oops/methodData.hpp"
64 #include "oops/objArrayKlass.inline.hpp"
65 #include "oops/oop.inline.hpp"
66 #include "oops/typeArrayOop.inline.hpp"
67 #include "runtime/prefetch.inline.hpp"
68 #include "runtime/threads.hpp"
69 #include "utilities/align.hpp"
70 #include "utilities/copy.hpp"
71 #include "utilities/events.hpp"
72 #include "utilities/stack.inline.hpp"
73
74 Stack<oop, mtGC> SerialFullGC::_marking_stack;
75 Stack<ObjArrayTask, mtGC> SerialFullGC::_objarray_stack;
76
77 PreservedMarksSet SerialFullGC::_preserved_overflow_stack_set(false /* in_c_heap */);
78 size_t SerialFullGC::_preserved_count = 0;
79 size_t SerialFullGC::_preserved_count_max = 0;
80 PreservedMark* SerialFullGC::_preserved_marks = nullptr;
81 STWGCTimer* SerialFullGC::_gc_timer = nullptr;
82 SerialOldTracer* SerialFullGC::_gc_tracer = nullptr;
83
84 AlwaysTrueClosure SerialFullGC::_always_true_closure;
85 ReferenceProcessor* SerialFullGC::_ref_processor;
86
87 StringDedup::Requests* SerialFullGC::_string_dedup_requests = nullptr;
88
89 SerialFullGC::FollowRootClosure SerialFullGC::follow_root_closure;
90
91 MarkAndPushClosure SerialFullGC::mark_and_push_closure(ClassLoaderData::_claim_stw_fullgc_mark);
92 CLDToOopClosure SerialFullGC::follow_cld_closure(&mark_and_push_closure, ClassLoaderData::_claim_stw_fullgc_mark);
93 CLDToOopClosure SerialFullGC::adjust_cld_closure(&adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
94
95 class DeadSpacer : StackObj {
96 size_t _allowed_deadspace_words;
97 bool _active;
98 ContiguousSpace* _space;
99
100 public:
101 DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
102 size_t ratio = (_space == SerialHeap::heap()->old_gen()->space())
103 ? MarkSweepDeadRatio : 0;
104 _active = ratio > 0;
105
106 if (_active) {
107 // We allow some amount of garbage towards the bottom of the space, so
108 // we don't start compacting before there is a significant gain to be made.
109 // Occasionally, we want to ensure a full compaction, which is determined
110 // by the MarkSweepAlwaysCompactCount parameter.
111 if ((SerialHeap::heap()->total_full_collections() % MarkSweepAlwaysCompactCount) != 0) {
112 _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
113 } else {
114 _active = false;
115 }
116 }
117 }
118
119 bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
120 if (!_active) {
121 return false;
122 }
123
124 size_t dead_length = pointer_delta(dead_end, dead_start);
125 if (_allowed_deadspace_words >= dead_length) {
126 _allowed_deadspace_words -= dead_length;
127 CollectedHeap::fill_with_object(dead_start, dead_length);
128 oop obj = cast_to_oop(dead_start);
129 // obj->set_mark(obj->mark().set_marked());
130
131 assert(dead_length == obj->size(), "bad filler object size");
132 log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", %zub",
133 p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
134
135 return true;
136 } else {
137 _active = false;
138 return false;
139 }
140 }
141 };
142
143 // Implement the "compaction" part of the mark-compact GC algorithm.
144 class Compacter {
145 // There are four spaces in total, but only the first three can be used after
146 // compact. IOW, old and eden/from must be enough for all live objs
147 static constexpr uint max_num_spaces = 4;
148
149 struct CompactionSpace {
150 ContiguousSpace* _space;
151 // Will be the new top after compaction is complete.
152 HeapWord* _compaction_top;
153 // The first dead word in this contiguous space. It's an optimization to
154 // skip large chunk of live objects at the beginning.
155 HeapWord* _first_dead;
156
157 void init(ContiguousSpace* space) {
158 _space = space;
159 _compaction_top = space->bottom();
160 _first_dead = nullptr;
161 }
162 };
163
164 CompactionSpace _spaces[max_num_spaces];
165 // The num of spaces to be compacted, i.e. containing live objs.
166 uint _num_spaces;
167
168 uint _index;
169
170 // Used for BOT update
171 TenuredGeneration* _old_gen;
172
173 HeapWord* get_compaction_top(uint index) const {
174 return _spaces[index]._compaction_top;
175 }
176
177 HeapWord* get_first_dead(uint index) const {
178 return _spaces[index]._first_dead;
179 }
180
181 ContiguousSpace* get_space(uint index) const {
182 return _spaces[index]._space;
183 }
184
185 void record_first_dead(uint index, HeapWord* first_dead) {
186 assert(_spaces[index]._first_dead == nullptr, "should write only once");
187 _spaces[index]._first_dead = first_dead;
188 }
189
190 HeapWord* alloc(size_t old_size, size_t new_size, HeapWord* old_obj) {
191 size_t words = (old_obj == _spaces[_index]._compaction_top) ? old_size : new_size;
192 while (true) {
193 if (words <= pointer_delta(_spaces[_index]._space->end(),
194 _spaces[_index]._compaction_top)) {
195 HeapWord* result = _spaces[_index]._compaction_top;
196 _spaces[_index]._compaction_top += words;
197 if (_index == 0) {
198 // old-gen requires BOT update
199 _old_gen->update_for_block(result, result + words);
200 }
201 return result;
202 }
203
204 // out-of-memory in this space
205 _index++;
206 assert(_index < max_num_spaces - 1, "the last space should not be used");
207 words = (old_obj == _spaces[_index]._compaction_top) ? old_size : new_size;
208 }
209 }
210
211 static void prefetch_read_scan(void* p) {
212 if (PrefetchScanIntervalInBytes >= 0) {
213 Prefetch::read(p, PrefetchScanIntervalInBytes);
214 }
215 }
216
217 static void prefetch_write_scan(void* p) {
218 if (PrefetchScanIntervalInBytes >= 0) {
219 Prefetch::write(p, PrefetchScanIntervalInBytes);
220 }
221 }
222
223 static void prefetch_write_copy(void* p) {
224 if (PrefetchCopyIntervalInBytes >= 0) {
225 Prefetch::write(p, PrefetchCopyIntervalInBytes);
226 }
227 }
228
229 static void forward_obj(oop obj, HeapWord* new_addr, bool after_first_dead) {
230 prefetch_write_scan(obj);
231 if (cast_from_oop<HeapWord*>(obj) != new_addr) {
232 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
233 } else {
234 assert(obj->is_gc_marked(), "inv");
235 if (!after_first_dead) {
236 // This obj will stay in-place and we'll not see it during relocation.
237 // Fix the markword.
238 obj->reinit_mark();
239 } else {
240 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
241 }
242 }
243 }
244
245 static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
246 for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
247 prefetch_read_scan(i_addr);
248 oop obj = cast_to_oop(i_addr);
249 if (obj->is_gc_marked()) {
250 return i_addr;
251 }
252 i_addr += obj->size();
253 }
254 return end;
255 };
256
257 static size_t relocate(HeapWord* addr) {
258 // Prefetch source and destination
259 prefetch_read_scan(addr);
260
261 oop obj = cast_to_oop(addr);
262 oop new_obj = FullGCForwarding::forwardee(obj);
263 HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
264
265 size_t obj_size = obj->size();
266 if (addr != new_addr) {
267 prefetch_write_copy(new_addr);
268 Copy::aligned_conjoint_words(addr, new_addr, obj_size);
269 }
270 new_obj->reinit_mark();
271 if (addr != new_addr) {
272 new_obj->initialize_hash_if_necessary(obj);
273 }
274
275 return obj_size;
276 }
277
278 public:
279 explicit Compacter(SerialHeap* heap) {
280 // In this order so that heap is compacted towards old-gen.
281 _spaces[0].init(heap->old_gen()->space());
282 _spaces[1].init(heap->young_gen()->eden());
283 _spaces[2].init(heap->young_gen()->from());
284
285 bool is_promotion_failed = !heap->young_gen()->to()->is_empty();
286 if (is_promotion_failed) {
287 _spaces[3].init(heap->young_gen()->to());
288 _num_spaces = 4;
289 } else {
290 _num_spaces = 3;
291 }
292 _index = 0;
293 _old_gen = heap->old_gen();
294 }
295
296 void phase2_calculate_new_addr() {
297 for (uint i = 0; i < _num_spaces; ++i) {
298 ContiguousSpace* space = get_space(i);
299 HeapWord* cur_addr = space->bottom();
300 HeapWord* top = space->top();
301
302 bool record_first_dead_done = false;
303
304 DeadSpacer dead_spacer(space);
305
306 while (cur_addr < top) {
307 oop obj = cast_to_oop(cur_addr);
308 size_t obj_size = obj->size();
309 size_t new_size = obj->copy_size(obj_size, obj->mark());
310 if (obj->is_gc_marked()) {
311 HeapWord* new_addr = alloc(obj_size, new_size, cur_addr);
312 forward_obj(obj, new_addr, record_first_dead_done);
313 assert(obj->size() == obj_size, "size must not change after forwarding");
314 cur_addr += obj_size;
315 } else {
316 // Skipping the current known-unmarked obj
317 HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
318 if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
319 // Register space for the filler obj
320 size_t size = pointer_delta(next_live_addr, cur_addr);
321 alloc(size, size, cur_addr);
322 } else {
323 if (!record_first_dead_done) {
324 record_first_dead(i, cur_addr);
325 record_first_dead_done = true;
326 }
327 *(HeapWord**)cur_addr = next_live_addr;
328 }
329 cur_addr = next_live_addr;
330 }
331 }
332
333 if (!record_first_dead_done) {
334 record_first_dead(i, top);
335 }
336 }
337 }
338
339 void phase3_adjust_pointers() {
340 for (uint i = 0; i < _num_spaces; ++i) {
341 ContiguousSpace* space = get_space(i);
342 HeapWord* cur_addr = space->bottom();
343 HeapWord* const top = space->top();
344 HeapWord* const first_dead = get_first_dead(i);
345
346 while (cur_addr < top) {
347 prefetch_write_scan(cur_addr);
348 if (cur_addr < first_dead || cast_to_oop(cur_addr)->is_gc_marked()) {
349 size_t size = cast_to_oop(cur_addr)->oop_iterate_size(&SerialFullGC::adjust_pointer_closure);
350 cur_addr += size;
351 } else {
352 assert(*(HeapWord**)cur_addr > cur_addr, "forward progress");
353 cur_addr = *(HeapWord**)cur_addr;
354 }
355 }
356 }
357 }
358
359 void phase4_compact() {
360 for (uint i = 0; i < _num_spaces; ++i) {
361 ContiguousSpace* space = get_space(i);
362 HeapWord* cur_addr = space->bottom();
363 HeapWord* top = space->top();
364
365 // Check if the first obj inside this space is forwarded.
366 if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
367 // Jump over consecutive (in-place) live-objs-chunk
368 cur_addr = get_first_dead(i);
369 }
370
371 while (cur_addr < top) {
372 if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
373 cur_addr = *(HeapWord**) cur_addr;
374 continue;
375 }
376 cur_addr += relocate(cur_addr);
377 }
378
379 // Reset top and unused memory
380 HeapWord* new_top = get_compaction_top(i);
381 space->set_top(new_top);
382 if (ZapUnusedHeapArea && new_top < top) {
383 space->mangle_unused_area(MemRegion(new_top, top));
384 }
385 }
386 }
387 };
388
389 template <class T> void SerialFullGC::KeepAliveClosure::do_oop_work(T* p) {
390 mark_and_push(p);
391 }
392
393 void SerialFullGC::push_objarray(oop obj, size_t index) {
394 ObjArrayTask task(obj, index);
395 assert(task.is_valid(), "bad ObjArrayTask");
396 _objarray_stack.push(task);
397 }
398
399 void SerialFullGC::follow_array(objArrayOop array) {
400 mark_and_push_closure.do_klass(array->klass());
401 // Don't push empty arrays to avoid unnecessary work.
402 if (array->length() > 0) {
403 SerialFullGC::push_objarray(array, 0);
404 }
405 }
406
407 void SerialFullGC::follow_object(oop obj) {
408 assert(obj->is_gc_marked(), "should be marked");
409 if (obj->is_objArray()) {
410 // Handle object arrays explicitly to allow them to
411 // be split into chunks if needed.
412 SerialFullGC::follow_array((objArrayOop)obj);
413 } else {
414 obj->oop_iterate(&mark_and_push_closure);
415 }
416 }
417
418 void SerialFullGC::follow_array_chunk(objArrayOop array, int index) {
419 const int len = array->length();
420 const int beg_index = index;
421 assert(beg_index < len || len == 0, "index too large");
422
423 const int stride = MIN2(len - beg_index, (int) ObjArrayMarkingStride);
424 const int end_index = beg_index + stride;
425
426 array->oop_iterate_elements_range(&mark_and_push_closure, beg_index, end_index);
427
428 if (end_index < len) {
429 SerialFullGC::push_objarray(array, end_index); // Push the continuation.
430 }
431 }
432
433 void SerialFullGC::follow_stack() {
434 do {
435 while (!_marking_stack.is_empty()) {
436 oop obj = _marking_stack.pop();
437 assert (obj->is_gc_marked(), "p must be marked");
438 follow_object(obj);
439 }
440 // Process ObjArrays one at a time to avoid marking stack bloat.
441 if (!_objarray_stack.is_empty()) {
442 ObjArrayTask task = _objarray_stack.pop();
443 follow_array_chunk(objArrayOop(task.obj()), task.index());
444 }
445 } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty());
446 }
447
448 SerialFullGC::FollowStackClosure SerialFullGC::follow_stack_closure;
449
450 void SerialFullGC::FollowStackClosure::do_void() { follow_stack(); }
451
452 template <class T> void SerialFullGC::follow_root(T* p) {
453 assert(!Universe::heap()->is_in(p),
454 "roots shouldn't be things within the heap");
455 T heap_oop = RawAccess<>::oop_load(p);
456 if (!CompressedOops::is_null(heap_oop)) {
457 oop obj = CompressedOops::decode_not_null(heap_oop);
458 if (!obj->mark().is_marked()) {
459 mark_object(obj);
460 follow_object(obj);
461 }
462 }
463 follow_stack();
464 }
465
466 void SerialFullGC::FollowRootClosure::do_oop(oop* p) { follow_root(p); }
467 void SerialFullGC::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); }
468
469 // We preserve the mark which should be replaced at the end and the location
470 // that it will go. Note that the object that this markWord belongs to isn't
471 // currently at that address but it will be after phase4
472 void SerialFullGC::preserve_mark(oop obj, markWord mark) {
473 // We try to store preserved marks in the to space of the new generation since
474 // this is storage which should be available. Most of the time this should be
475 // sufficient space for the marks we need to preserve but if it isn't we fall
476 // back to using Stacks to keep track of the overflow.
477 if (_preserved_count < _preserved_count_max) {
478 _preserved_marks[_preserved_count++] = PreservedMark(obj, mark);
479 } else {
480 _preserved_overflow_stack_set.get()->push_always(obj, mark);
481 }
482 }
483
484 void SerialFullGC::phase1_mark(bool clear_all_softrefs) {
485 // Recursively traverse all live objects and mark them
486 GCTraceTime(Info, gc, phases) tm("Phase 1: Mark live objects", _gc_timer);
487
488 SerialHeap* gch = SerialHeap::heap();
489
490 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_mark);
491
492 ref_processor()->start_discovery(clear_all_softrefs);
493
494 {
495 GCTraceTime(Debug, gc, phases) tm_m("Marking From Roots", gc_timer());
496
497 // Start tracing from roots, there are 3 kinds of roots in full-gc.
498 //
499 // 1. CLD. This method internally takes care of whether class loading is
500 // enabled or not, applying the closure to both strong and weak or only
501 // strong CLDs.
502 ClassLoaderDataGraph::always_strong_cld_do(&follow_cld_closure);
503
504 {
505 // 2. Threads stack frames and active nmethods in them.
506 NMethodMarkingScope nmethod_marking_scope;
507 MarkingNMethodClosure mark_code_closure(&follow_root_closure);
508
509 Threads::oops_do(&follow_root_closure, &mark_code_closure);
510 }
511
512 // 3. VM internal roots.
513 OopStorageSet::strong_oops_do(&follow_root_closure);
514 }
515
516 // Process reference objects found during marking
517 {
518 GCTraceTime(Debug, gc, phases) tm_m("Reference Processing", gc_timer());
519
520 ReferenceProcessorPhaseTimes pt(_gc_timer, ref_processor()->max_num_queues());
521 SerialGCRefProcProxyTask task(is_alive, keep_alive, follow_stack_closure);
522 const ReferenceProcessorStats& stats = ref_processor()->process_discovered_references(task, nullptr, pt);
523 pt.print_all_references();
524 gc_tracer()->report_gc_reference_stats(stats);
525 }
526
527 // This is the point where the entire marking should have completed.
528 assert(_marking_stack.is_empty(), "Marking should have completed");
529
530 {
531 GCTraceTime(Debug, gc, phases) tm_m("Weak Processing", gc_timer());
532 WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl);
533 }
534
535 {
536 GCTraceTime(Debug, gc, phases) tm_m("Class Unloading", gc_timer());
537
538 ClassUnloadingContext* ctx = ClassUnloadingContext::context();
539
540 bool unloading_occurred;
541 {
542 CodeCache::UnlinkingScope scope(&is_alive);
543
544 // Unload classes and purge the SystemDictionary.
545 unloading_occurred = SystemDictionary::do_unloading(gc_timer());
546
547 // Unload nmethods.
548 CodeCache::do_unloading(unloading_occurred);
549 }
550
551 {
552 GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", gc_timer());
553 // Release unloaded nmethod's memory.
554 ctx->purge_nmethods();
555 }
556 {
557 GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", gc_timer());
558 gch->prune_unlinked_nmethods();
559 }
560 {
561 GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
562 ctx->free_nmethods();
563 }
564
565 // Prune dead klasses from subklass/sibling/implementor lists.
566 Klass::clean_weak_klass_links(unloading_occurred);
567 }
568
569 {
570 GCTraceTime(Debug, gc, phases) tm_m("Report Object Count", gc_timer());
571 gc_tracer()->report_object_count_after_gc(&is_alive, nullptr);
572 }
573 }
574
575 void SerialFullGC::allocate_stacks() {
576 void* scratch = nullptr;
577 size_t num_words;
578 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
579 young_gen->contribute_scratch(scratch, num_words);
580
581 if (scratch != nullptr) {
582 _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
583 } else {
584 _preserved_count_max = 0;
585 }
586
587 _preserved_marks = (PreservedMark*)scratch;
588 _preserved_count = 0;
589
590 _preserved_overflow_stack_set.init(1);
591 }
592
593 void SerialFullGC::deallocate_stacks() {
594 if (_preserved_count_max != 0) {
595 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
596 young_gen->reset_scratch();
597 }
598
599 _preserved_overflow_stack_set.reclaim();
600 _marking_stack.clear();
601 _objarray_stack.clear(true);
602 }
603
604 void SerialFullGC::mark_object(oop obj) {
605 if (StringDedup::is_enabled() &&
606 java_lang_String::is_instance(obj) &&
607 SerialStringDedup::is_candidate_from_mark(obj)) {
608 _string_dedup_requests->add(obj);
609 }
610
611 // some marks may contain information we need to preserve so we store them away
612 // and overwrite the mark. We'll restore it at the end of serial full GC.
613 markWord mark = obj->mark();
614 obj->set_mark(mark.set_marked());
615
616 ContinuationGCSupport::transform_stack_chunk(obj);
617
618 if (obj->mark_must_be_preserved(mark)) {
619 preserve_mark(obj, mark);
620 }
621 }
622
623 template <class T> void SerialFullGC::mark_and_push(T* p) {
624 T heap_oop = RawAccess<>::oop_load(p);
625 if (!CompressedOops::is_null(heap_oop)) {
626 oop obj = CompressedOops::decode_not_null(heap_oop);
627 if (!obj->mark().is_marked()) {
628 mark_object(obj);
629 _marking_stack.push(obj);
630 }
631 }
632 }
633
634 template <typename T>
635 void MarkAndPushClosure::do_oop_work(T* p) { SerialFullGC::mark_and_push(p); }
636 void MarkAndPushClosure::do_oop( oop* p) { do_oop_work(p); }
637 void MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_work(p); }
638
639 template <class T> void SerialFullGC::adjust_pointer(T* p) {
640 T heap_oop = RawAccess<>::oop_load(p);
641 if (!CompressedOops::is_null(heap_oop)) {
642 oop obj = CompressedOops::decode_not_null(heap_oop);
643 assert(Universe::heap()->is_in(obj), "should be in heap");
644
645 if (FullGCForwarding::is_forwarded(obj)) {
646 oop new_obj = FullGCForwarding::forwardee(obj);
647 assert(is_object_aligned(new_obj), "oop must be aligned");
648 RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
649 }
650 }
651 }
652
653 template <typename T>
654 void AdjustPointerClosure::do_oop_work(T* p) { SerialFullGC::adjust_pointer(p); }
655 inline void AdjustPointerClosure::do_oop(oop* p) { do_oop_work(p); }
656 inline void AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_work(p); }
657
658 AdjustPointerClosure SerialFullGC::adjust_pointer_closure;
659
660 void SerialFullGC::adjust_marks() {
661 // adjust the oops we saved earlier
662 for (size_t i = 0; i < _preserved_count; i++) {
663 PreservedMarks::adjust_preserved_mark(_preserved_marks + i);
664 }
665
666 // deal with the overflow stack
667 _preserved_overflow_stack_set.get()->adjust_during_full_gc();
668 }
669
670 void SerialFullGC::restore_marks() {
671 log_trace(gc)("Restoring %zu marks", _preserved_count + _preserved_overflow_stack_set.get()->size());
672
673 // restore the marks we saved earlier
674 for (size_t i = 0; i < _preserved_count; i++) {
675 _preserved_marks[i].set_mark();
676 }
677
678 // deal with the overflow
679 _preserved_overflow_stack_set.restore(nullptr);
680 }
681
682 SerialFullGC::IsAliveClosure SerialFullGC::is_alive;
683
684 bool SerialFullGC::IsAliveClosure::do_object_b(oop p) { return p->is_gc_marked(); }
685
686 SerialFullGC::KeepAliveClosure SerialFullGC::keep_alive;
687
688 void SerialFullGC::KeepAliveClosure::do_oop(oop* p) { SerialFullGC::KeepAliveClosure::do_oop_work(p); }
689 void SerialFullGC::KeepAliveClosure::do_oop(narrowOop* p) { SerialFullGC::KeepAliveClosure::do_oop_work(p); }
690
691 void SerialFullGC::initialize() {
692 SerialFullGC::_gc_timer = new STWGCTimer();
693 SerialFullGC::_gc_tracer = new SerialOldTracer();
694 SerialFullGC::_string_dedup_requests = new StringDedup::Requests();
695
696 // The Full GC operates on the entire heap so all objects should be subject
697 // to discovery, hence the _always_true_closure.
698 SerialFullGC::_ref_processor = new ReferenceProcessor(&_always_true_closure);
699 mark_and_push_closure.set_ref_discoverer(_ref_processor);
700 }
701
702 void SerialFullGC::invoke_at_safepoint(bool clear_all_softrefs) {
703 assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
704
705 SerialHeap* gch = SerialHeap::heap();
706
707 gch->trace_heap_before_gc(_gc_tracer);
708
709 // Capture used regions for old-gen to reestablish old-to-young invariant
710 // after full-gc.
711 gch->old_gen()->save_used_region();
712
713 allocate_stacks();
714
715 // Usually, all class unloading work occurs at the end of phase 1, but Serial
716 // full-gc accesses dead-objs' klass to find out the start of next live-obj
717 // during phase 2. This requires klasses of dead-objs to be kept loaded.
718 // Therefore, we declare ClassUnloadingContext at the same level as
719 // full-gc phases, and purge dead classes (invoking
720 // ClassLoaderDataGraph::purge) after all phases of full-gc.
721 ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
722 false /* unregister_nmethods_during_purge */,
723 false /* lock_nmethod_free_separately */);
724
725 phase1_mark(clear_all_softrefs);
726
727 FullGCForwarding::begin();
728
729 Compacter compacter{gch};
730
731 {
732 // Now all live objects are marked, compute the new object addresses.
733 GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
734
735 compacter.phase2_calculate_new_addr();
736 }
737
738 // Don't add any more derived pointers during phase3
739 #ifdef COMPILER2
740 assert(DerivedPointerTable::is_active(), "Sanity");
741 DerivedPointerTable::set_active(false);
742 #endif // COMPILER2
743
744 {
745 // Adjust the pointers to reflect the new locations
746 GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
747
748 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
749
750 // Remap strong and weak roots in adjust phase.
751 // 1. All (strong and weak) CLDs.
752 ClassLoaderDataGraph::cld_do(&adjust_cld_closure);
753
754 // 2. Threads stack frames. No need to visit on-stack nmethods, because all
755 // nmethods are visited in one go via CodeCache::nmethods_do.
756 Threads::oops_do(&adjust_pointer_closure, nullptr);
757 NMethodToOopClosure nmethod_cl(&adjust_pointer_closure, NMethodToOopClosure::FixRelocations);
758 CodeCache::nmethods_do(&nmethod_cl);
759
760 // 3. VM internal roots
761 OopStorageSet::strong_oops_do(&adjust_pointer_closure);
762
763 // 4. VM internal weak roots
764 WeakProcessor::oops_do(&adjust_pointer_closure);
765
766 adjust_marks();
767 compacter.phase3_adjust_pointers();
768 }
769
770 {
771 // All pointers are now adjusted, move objects accordingly
772 GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
773
774 compacter.phase4_compact();
775 }
776
777 // Delete metaspaces for unloaded class loaders and clean up CLDG.
778 ClassLoaderDataGraph::purge(true /* at_safepoint */);
779 DEBUG_ONLY(MetaspaceUtils::verify();)
780
781 // Need to clear claim bits for the next full-gc (specifically phase 1 and 3).
782 ClassLoaderDataGraph::clear_claimed_marks();
783
784 restore_marks();
785
786 FullGCForwarding::end();
787
788 deallocate_stacks();
789
790 SerialFullGC::_string_dedup_requests->flush();
791
792 bool is_young_gen_empty = (gch->young_gen()->used() == 0);
793 gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
794
795 gch->prune_scavengable_nmethods();
796
797 // Update heap occupancy information which is used as
798 // input to soft ref clearing policy at the next gc.
799 Universe::heap()->update_capacity_and_used_at_gc();
800
801 // Signal that we have completed a visit to all live objects.
802 Universe::heap()->record_whole_heap_examined_timestamp();
803
804 gch->trace_heap_after_gc(_gc_tracer);
805 }