1 /*
2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classLoaderDataGraph.hpp"
27 #include "classfile/classLoaderData.inline.hpp"
28 #include "classfile/javaClasses.inline.hpp"
29 #include "classfile/stringTable.hpp"
30 #include "classfile/symbolTable.hpp"
31 #include "classfile/systemDictionary.hpp"
32 #include "classfile/vmSymbols.hpp"
33 #include "code/codeCache.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/serial/cardTableRS.hpp"
37 #include "gc/serial/defNewGeneration.hpp"
38 #include "gc/serial/serialFullGC.hpp"
39 #include "gc/serial/serialGcRefProcProxyTask.hpp"
40 #include "gc/serial/serialHeap.hpp"
41 #include "gc/serial/serialStringDedup.hpp"
42 #include "gc/shared/classUnloadingContext.hpp"
43 #include "gc/shared/collectedHeap.inline.hpp"
44 #include "gc/shared/continuationGCSupport.inline.hpp"
45 #include "gc/shared/gcHeapSummary.hpp"
46 #include "gc/shared/gcTimer.hpp"
47 #include "gc/shared/gcTrace.hpp"
48 #include "gc/shared/gcTraceTime.inline.hpp"
49 #include "gc/shared/gc_globals.hpp"
50 #include "gc/shared/modRefBarrierSet.hpp"
51 #include "gc/shared/preservedMarks.inline.hpp"
52 #include "gc/shared/referencePolicy.hpp"
53 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
54 #include "gc/shared/slidingForwarding.inline.hpp"
55 #include "gc/shared/space.inline.hpp"
56 #include "gc/shared/strongRootsScope.hpp"
57 #include "gc/shared/weakProcessor.hpp"
58 #include "memory/iterator.inline.hpp"
59 #include "memory/universe.hpp"
60 #include "oops/access.inline.hpp"
61 #include "oops/compressedOops.inline.hpp"
62 #include "oops/instanceRefKlass.hpp"
63 #include "oops/markWord.hpp"
64 #include "oops/methodData.hpp"
65 #include "oops/objArrayKlass.inline.hpp"
66 #include "oops/oop.inline.hpp"
67 #include "oops/typeArrayOop.inline.hpp"
68 #include "runtime/prefetch.inline.hpp"
69 #include "utilities/align.hpp"
70 #include "utilities/copy.hpp"
71 #include "utilities/events.hpp"
72 #include "utilities/stack.inline.hpp"
73 #if INCLUDE_JVMCI
74 #include "jvmci/jvmci.hpp"
75 #endif
76
77 uint SerialFullGC::_total_invocations = 0;
78
79 Stack<oop, mtGC> SerialFullGC::_marking_stack;
80 Stack<ObjArrayTask, mtGC> SerialFullGC::_objarray_stack;
81
82 PreservedMarksSet SerialFullGC::_preserved_overflow_stack_set(false /* in_c_heap */);
83 size_t SerialFullGC::_preserved_count = 0;
84 size_t SerialFullGC::_preserved_count_max = 0;
85 PreservedMark* SerialFullGC::_preserved_marks = nullptr;
86 STWGCTimer* SerialFullGC::_gc_timer = nullptr;
87 SerialOldTracer* SerialFullGC::_gc_tracer = nullptr;
88
89 AlwaysTrueClosure SerialFullGC::_always_true_closure;
90 ReferenceProcessor* SerialFullGC::_ref_processor;
91
92 StringDedup::Requests* SerialFullGC::_string_dedup_requests = nullptr;
93
94 SerialFullGC::FollowRootClosure SerialFullGC::follow_root_closure;
95
96 MarkAndPushClosure SerialFullGC::mark_and_push_closure(ClassLoaderData::_claim_stw_fullgc_mark);
97 CLDToOopClosure SerialFullGC::follow_cld_closure(&mark_and_push_closure, ClassLoaderData::_claim_stw_fullgc_mark);
98 CLDToOopClosure SerialFullGC::adjust_cld_closure(&adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
99
100 class DeadSpacer : StackObj {
101 size_t _allowed_deadspace_words;
102 bool _active;
103 ContiguousSpace* _space;
104
105 public:
106 DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
107 size_t ratio = (_space == SerialHeap::heap()->old_gen()->space())
108 ? MarkSweepDeadRatio : 0;
109 _active = ratio > 0;
110
111 if (_active) {
112 // We allow some amount of garbage towards the bottom of the space, so
113 // we don't start compacting before there is a significant gain to be made.
114 // Occasionally, we want to ensure a full compaction, which is determined
115 // by the MarkSweepAlwaysCompactCount parameter.
116 if ((SerialFullGC::total_invocations() % MarkSweepAlwaysCompactCount) != 0) {
117 _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
118 } else {
119 _active = false;
120 }
121 }
122 }
123
124 bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
125 if (!_active) {
126 return false;
127 }
128
129 size_t dead_length = pointer_delta(dead_end, dead_start);
130 if (_allowed_deadspace_words >= dead_length) {
131 _allowed_deadspace_words -= dead_length;
132 CollectedHeap::fill_with_object(dead_start, dead_length);
133 oop obj = cast_to_oop(dead_start);
134 // obj->set_mark(obj->mark().set_marked());
135
136 assert(dead_length == obj->size(), "bad filler object size");
137 log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b",
138 p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
139
140 return true;
141 } else {
142 _active = false;
143 return false;
144 }
145 }
146 };
147
148 // Implement the "compaction" part of the mark-compact GC algorithm.
149 class Compacter {
150 // There are four spaces in total, but only the first three can be used after
151 // compact. IOW, old and eden/from must be enough for all live objs
152 static constexpr uint max_num_spaces = 4;
153
154 struct CompactionSpace {
155 ContiguousSpace* _space;
156 // Will be the new top after compaction is complete.
157 HeapWord* _compaction_top;
158 // The first dead word in this contiguous space. It's an optimization to
159 // skip large chunk of live objects at the beginning.
160 HeapWord* _first_dead;
161
162 void init(ContiguousSpace* space) {
163 _space = space;
164 _compaction_top = space->bottom();
165 _first_dead = nullptr;
166 }
167 };
168
169 CompactionSpace _spaces[max_num_spaces];
170 // The num of spaces to be compacted, i.e. containing live objs.
171 uint _num_spaces;
172
173 uint _index;
174
175 HeapWord* get_compaction_top(uint index) const {
176 return _spaces[index]._compaction_top;
177 }
178
179 HeapWord* get_first_dead(uint index) const {
180 return _spaces[index]._first_dead;
181 }
182
183 ContiguousSpace* get_space(uint index) const {
184 return _spaces[index]._space;
185 }
186
187 void record_first_dead(uint index, HeapWord* first_dead) {
188 assert(_spaces[index]._first_dead == nullptr, "should write only once");
189 _spaces[index]._first_dead = first_dead;
190 }
191
192 HeapWord* alloc(size_t words) {
193 while (true) {
194 if (words <= pointer_delta(_spaces[_index]._space->end(),
195 _spaces[_index]._compaction_top)) {
196 HeapWord* result = _spaces[_index]._compaction_top;
197 _spaces[_index]._compaction_top += words;
198 if (_index == 0) {
199 // old-gen requires BOT update
200 static_cast<TenuredSpace*>(_spaces[0]._space)->update_for_block(result, result + words);
201 }
202 return result;
203 }
204
205 // out-of-memory in this space
206 _index++;
207 assert(_index < max_num_spaces - 1, "the last space should not be used");
208 }
209 }
210
211 static void prefetch_read_scan(void* p) {
212 if (PrefetchScanIntervalInBytes >= 0) {
213 Prefetch::read(p, PrefetchScanIntervalInBytes);
214 }
215 }
216
217 static void prefetch_write_scan(void* p) {
218 if (PrefetchScanIntervalInBytes >= 0) {
219 Prefetch::write(p, PrefetchScanIntervalInBytes);
220 }
221 }
222
223 static void prefetch_write_copy(void* p) {
224 if (PrefetchCopyIntervalInBytes >= 0) {
225 Prefetch::write(p, PrefetchCopyIntervalInBytes);
226 }
227 }
228
229 static void forward_obj(oop obj, HeapWord* new_addr) {
230 prefetch_write_scan(obj);
231 if (cast_from_oop<HeapWord*>(obj) != new_addr) {
232 SlidingForwarding::forward_to(obj, cast_to_oop(new_addr));
233 } else {
234 assert(obj->is_gc_marked(), "inv");
235 // This obj will stay in-place. Fix the markword.
236 obj->init_mark();
237 }
238 }
239
240 static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
241 for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
242 prefetch_read_scan(i_addr);
243 oop obj = cast_to_oop(i_addr);
244 if (obj->is_gc_marked()) {
245 return i_addr;
246 }
247 i_addr += obj->size();
248 }
249 return end;
250 };
251
252 static size_t relocate(HeapWord* addr) {
253 // Prefetch source and destination
254 prefetch_read_scan(addr);
255
256 oop obj = cast_to_oop(addr);
257 oop new_obj = SlidingForwarding::forwardee(obj);
258 HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
259 assert(addr != new_addr, "inv");
260 prefetch_write_copy(new_addr);
261
262 size_t obj_size = obj->size();
263 Copy::aligned_conjoint_words(addr, new_addr, obj_size);
264 new_obj->init_mark();
265
266 return obj_size;
267 }
268
269 public:
270 explicit Compacter(SerialHeap* heap) {
271 // In this order so that heap is compacted towards old-gen.
272 _spaces[0].init(heap->old_gen()->space());
273 _spaces[1].init(heap->young_gen()->eden());
274 _spaces[2].init(heap->young_gen()->from());
275
276 bool is_promotion_failed = !heap->young_gen()->to()->is_empty();
277 if (is_promotion_failed) {
278 _spaces[3].init(heap->young_gen()->to());
279 _num_spaces = 4;
280 } else {
281 _num_spaces = 3;
282 }
283 _index = 0;
284 }
285
286 void phase2_calculate_new_addr() {
287 for (uint i = 0; i < _num_spaces; ++i) {
288 ContiguousSpace* space = get_space(i);
289 HeapWord* cur_addr = space->bottom();
290 HeapWord* top = space->top();
291
292 bool record_first_dead_done = false;
293
294 DeadSpacer dead_spacer(space);
295
296 while (cur_addr < top) {
297 oop obj = cast_to_oop(cur_addr);
298 size_t obj_size = obj->size();
299 if (obj->is_gc_marked()) {
300 HeapWord* new_addr = alloc(obj_size);
301 forward_obj(obj, new_addr);
302 cur_addr += obj_size;
303 } else {
304 // Skipping the current known-unmarked obj
305 HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
306 if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
307 // Register space for the filler obj
308 alloc(pointer_delta(next_live_addr, cur_addr));
309 } else {
310 if (!record_first_dead_done) {
311 record_first_dead(i, cur_addr);
312 record_first_dead_done = true;
313 }
314 *(HeapWord**)cur_addr = next_live_addr;
315 }
316 cur_addr = next_live_addr;
317 }
318 }
319
320 if (!record_first_dead_done) {
321 record_first_dead(i, top);
322 }
323 }
324 }
325
326 void phase3_adjust_pointers() {
327 for (uint i = 0; i < _num_spaces; ++i) {
328 ContiguousSpace* space = get_space(i);
329 HeapWord* cur_addr = space->bottom();
330 HeapWord* const top = space->top();
331 HeapWord* const first_dead = get_first_dead(i);
332
333 while (cur_addr < top) {
334 prefetch_write_scan(cur_addr);
335 if (cur_addr < first_dead || cast_to_oop(cur_addr)->is_gc_marked()) {
336 size_t size = cast_to_oop(cur_addr)->oop_iterate_size(&SerialFullGC::adjust_pointer_closure);
337 cur_addr += size;
338 } else {
339 assert(*(HeapWord**)cur_addr > cur_addr, "forward progress");
340 cur_addr = *(HeapWord**)cur_addr;
341 }
342 }
343 }
344 }
345
346 void phase4_compact() {
347 for (uint i = 0; i < _num_spaces; ++i) {
348 ContiguousSpace* space = get_space(i);
349 HeapWord* cur_addr = space->bottom();
350 HeapWord* top = space->top();
351
352 // Check if the first obj inside this space is forwarded.
353 if (SlidingForwarding::is_not_forwarded(cast_to_oop(cur_addr))) {
354 // Jump over consecutive (in-place) live-objs-chunk
355 cur_addr = get_first_dead(i);
356 }
357
358 while (cur_addr < top) {
359 if (SlidingForwarding::is_not_forwarded(cast_to_oop(cur_addr))) {
360 cur_addr = *(HeapWord**) cur_addr;
361 continue;
362 }
363 cur_addr += relocate(cur_addr);
364 }
365
366 // Reset top and unused memory
367 space->set_top(get_compaction_top(i));
368 if (ZapUnusedHeapArea) {
369 space->mangle_unused_area();
370 }
371 }
372 }
373 };
374
375 template <class T> void SerialFullGC::KeepAliveClosure::do_oop_work(T* p) {
376 mark_and_push(p);
377 }
378
379 void SerialFullGC::push_objarray(oop obj, size_t index) {
380 ObjArrayTask task(obj, index);
381 assert(task.is_valid(), "bad ObjArrayTask");
382 _objarray_stack.push(task);
383 }
384
385 void SerialFullGC::follow_array(objArrayOop array) {
386 mark_and_push_closure.do_klass(array->klass());
387 // Don't push empty arrays to avoid unnecessary work.
388 if (array->length() > 0) {
389 SerialFullGC::push_objarray(array, 0);
390 }
391 }
392
393 void SerialFullGC::follow_object(oop obj) {
394 assert(obj->is_gc_marked(), "should be marked");
395 if (obj->is_objArray()) {
396 // Handle object arrays explicitly to allow them to
397 // be split into chunks if needed.
398 SerialFullGC::follow_array((objArrayOop)obj);
399 } else {
400 obj->oop_iterate(&mark_and_push_closure);
401 }
402 }
403
404 void SerialFullGC::follow_array_chunk(objArrayOop array, int index) {
405 const int len = array->length();
406 const int beg_index = index;
407 assert(beg_index < len || len == 0, "index too large");
408
409 const int stride = MIN2(len - beg_index, (int) ObjArrayMarkingStride);
410 const int end_index = beg_index + stride;
411
412 array->oop_iterate_range(&mark_and_push_closure, beg_index, end_index);
413
414 if (end_index < len) {
415 SerialFullGC::push_objarray(array, end_index); // Push the continuation.
416 }
417 }
418
419 void SerialFullGC::follow_stack() {
420 do {
421 while (!_marking_stack.is_empty()) {
422 oop obj = _marking_stack.pop();
423 assert (obj->is_gc_marked(), "p must be marked");
424 follow_object(obj);
425 }
426 // Process ObjArrays one at a time to avoid marking stack bloat.
427 if (!_objarray_stack.is_empty()) {
428 ObjArrayTask task = _objarray_stack.pop();
429 follow_array_chunk(objArrayOop(task.obj()), task.index());
430 }
431 } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty());
432 }
433
434 SerialFullGC::FollowStackClosure SerialFullGC::follow_stack_closure;
435
436 void SerialFullGC::FollowStackClosure::do_void() { follow_stack(); }
437
438 template <class T> void SerialFullGC::follow_root(T* p) {
439 assert(!Universe::heap()->is_in(p),
440 "roots shouldn't be things within the heap");
441 T heap_oop = RawAccess<>::oop_load(p);
442 if (!CompressedOops::is_null(heap_oop)) {
443 oop obj = CompressedOops::decode_not_null(heap_oop);
444 if (!obj->mark().is_marked()) {
445 mark_object(obj);
446 follow_object(obj);
447 }
448 }
449 follow_stack();
450 }
451
452 void SerialFullGC::FollowRootClosure::do_oop(oop* p) { follow_root(p); }
453 void SerialFullGC::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); }
454
455 // We preserve the mark which should be replaced at the end and the location
456 // that it will go. Note that the object that this markWord belongs to isn't
457 // currently at that address but it will be after phase4
458 void SerialFullGC::preserve_mark(oop obj, markWord mark) {
459 // We try to store preserved marks in the to space of the new generation since
460 // this is storage which should be available. Most of the time this should be
461 // sufficient space for the marks we need to preserve but if it isn't we fall
462 // back to using Stacks to keep track of the overflow.
463 if (_preserved_count < _preserved_count_max) {
464 _preserved_marks[_preserved_count++] = PreservedMark(obj, mark);
465 } else {
466 _preserved_overflow_stack_set.get()->push_always(obj, mark);
467 }
468 }
469
470 void SerialFullGC::phase1_mark(bool clear_all_softrefs) {
471 // Recursively traverse all live objects and mark them
472 GCTraceTime(Info, gc, phases) tm("Phase 1: Mark live objects", _gc_timer);
473
474 SerialHeap* gch = SerialHeap::heap();
475
476 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_mark);
477
478 ref_processor()->start_discovery(clear_all_softrefs);
479
480 {
481 StrongRootsScope srs(0);
482
483 CLDClosure* weak_cld_closure = ClassUnloading ? nullptr : &follow_cld_closure;
484 MarkingNMethodClosure mark_code_closure(&follow_root_closure, !NMethodToOopClosure::FixRelocations, true);
485 gch->process_roots(SerialHeap::SO_None,
486 &follow_root_closure,
487 &follow_cld_closure,
488 weak_cld_closure,
489 &mark_code_closure);
490 }
491
492 // Process reference objects found during marking
493 {
494 GCTraceTime(Debug, gc, phases) tm_m("Reference Processing", gc_timer());
495
496 ReferenceProcessorPhaseTimes pt(_gc_timer, ref_processor()->max_num_queues());
497 SerialGCRefProcProxyTask task(is_alive, keep_alive, follow_stack_closure);
498 const ReferenceProcessorStats& stats = ref_processor()->process_discovered_references(task, pt);
499 pt.print_all_references();
500 gc_tracer()->report_gc_reference_stats(stats);
501 }
502
503 // This is the point where the entire marking should have completed.
504 assert(_marking_stack.is_empty(), "Marking should have completed");
505
506 {
507 GCTraceTime(Debug, gc, phases) tm_m("Weak Processing", gc_timer());
508 WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl);
509 }
510
511 {
512 GCTraceTime(Debug, gc, phases) tm_m("Class Unloading", gc_timer());
513
514 ClassUnloadingContext* ctx = ClassUnloadingContext::context();
515
516 bool unloading_occurred;
517 {
518 CodeCache::UnlinkingScope scope(&is_alive);
519
520 // Unload classes and purge the SystemDictionary.
521 unloading_occurred = SystemDictionary::do_unloading(gc_timer());
522
523 // Unload nmethods.
524 CodeCache::do_unloading(unloading_occurred);
525 }
526
527 {
528 GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", gc_timer());
529 // Release unloaded nmethod's memory.
530 ctx->purge_nmethods();
531 }
532 {
533 GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", gc_timer());
534 gch->prune_unlinked_nmethods();
535 }
536 {
537 GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
538 ctx->free_nmethods();
539 }
540
541 // Prune dead klasses from subklass/sibling/implementor lists.
542 Klass::clean_weak_klass_links(unloading_occurred);
543
544 // Clean JVMCI metadata handles.
545 JVMCI_ONLY(JVMCI::do_unloading(unloading_occurred));
546 }
547
548 {
549 GCTraceTime(Debug, gc, phases) tm_m("Report Object Count", gc_timer());
550 gc_tracer()->report_object_count_after_gc(&is_alive, nullptr);
551 }
552 }
553
554 void SerialFullGC::allocate_stacks() {
555 void* scratch = nullptr;
556 size_t num_words;
557 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
558 young_gen->contribute_scratch(scratch, num_words);
559
560 if (scratch != nullptr) {
561 _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
562 } else {
563 _preserved_count_max = 0;
564 }
565
566 _preserved_marks = (PreservedMark*)scratch;
567 _preserved_count = 0;
568
569 _preserved_overflow_stack_set.init(1);
570 }
571
572 void SerialFullGC::deallocate_stacks() {
573 if (_preserved_count_max != 0) {
574 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
575 young_gen->reset_scratch();
576 }
577
578 _preserved_overflow_stack_set.reclaim();
579 _marking_stack.clear();
580 _objarray_stack.clear(true);
581 }
582
583 void SerialFullGC::mark_object(oop obj) {
584 if (StringDedup::is_enabled() &&
585 java_lang_String::is_instance(obj) &&
586 SerialStringDedup::is_candidate_from_mark(obj)) {
587 _string_dedup_requests->add(obj);
588 }
589
590 // Do the transform while we still have the header intact,
591 // which might include important class information.
592 ContinuationGCSupport::transform_stack_chunk(obj);
593
594 // some marks may contain information we need to preserve so we store them away
595 // and overwrite the mark. We'll restore it at the end of serial full GC.
596 markWord mark = obj->mark();
597 obj->set_mark(obj->prototype_mark().set_marked());
598
599 if (obj->mark_must_be_preserved(mark)) {
600 preserve_mark(obj, mark);
601 }
602 }
603
604 template <class T> void SerialFullGC::mark_and_push(T* p) {
605 T heap_oop = RawAccess<>::oop_load(p);
606 if (!CompressedOops::is_null(heap_oop)) {
607 oop obj = CompressedOops::decode_not_null(heap_oop);
608 if (!obj->mark().is_marked()) {
609 mark_object(obj);
610 _marking_stack.push(obj);
611 }
612 }
613 }
614
615 template <typename T>
616 void MarkAndPushClosure::do_oop_work(T* p) { SerialFullGC::mark_and_push(p); }
617 void MarkAndPushClosure::do_oop( oop* p) { do_oop_work(p); }
618 void MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_work(p); }
619
620 template <class T> void SerialFullGC::adjust_pointer(T* p) {
621 T heap_oop = RawAccess<>::oop_load(p);
622 if (!CompressedOops::is_null(heap_oop)) {
623 oop obj = CompressedOops::decode_not_null(heap_oop);
624 assert(Universe::heap()->is_in(obj), "should be in heap");
625
626 if (SlidingForwarding::is_forwarded(obj)) {
627 oop new_obj = SlidingForwarding::forwardee(obj);
628 assert(is_object_aligned(new_obj), "oop must be aligned");
629 RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
630 }
631 }
632 }
633
634 template <typename T>
635 void AdjustPointerClosure::do_oop_work(T* p) { SerialFullGC::adjust_pointer(p); }
636 inline void AdjustPointerClosure::do_oop(oop* p) { do_oop_work(p); }
637 inline void AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_work(p); }
638
639 AdjustPointerClosure SerialFullGC::adjust_pointer_closure;
640
641 void SerialFullGC::adjust_marks() {
642 // adjust the oops we saved earlier
643 for (size_t i = 0; i < _preserved_count; i++) {
644 PreservedMarks::adjust_preserved_mark(_preserved_marks + i);
645 }
646
647 // deal with the overflow stack
648 _preserved_overflow_stack_set.get()->adjust_during_full_gc();
649 }
650
651 void SerialFullGC::restore_marks() {
652 log_trace(gc)("Restoring " SIZE_FORMAT " marks", _preserved_count + _preserved_overflow_stack_set.get()->size());
653
654 // restore the marks we saved earlier
655 for (size_t i = 0; i < _preserved_count; i++) {
656 _preserved_marks[i].set_mark();
657 }
658
659 // deal with the overflow
660 _preserved_overflow_stack_set.restore(nullptr);
661 }
662
663 SerialFullGC::IsAliveClosure SerialFullGC::is_alive;
664
665 bool SerialFullGC::IsAliveClosure::do_object_b(oop p) { return p->is_gc_marked(); }
666
667 SerialFullGC::KeepAliveClosure SerialFullGC::keep_alive;
668
669 void SerialFullGC::KeepAliveClosure::do_oop(oop* p) { SerialFullGC::KeepAliveClosure::do_oop_work(p); }
670 void SerialFullGC::KeepAliveClosure::do_oop(narrowOop* p) { SerialFullGC::KeepAliveClosure::do_oop_work(p); }
671
672 void SerialFullGC::initialize() {
673 SerialFullGC::_gc_timer = new STWGCTimer();
674 SerialFullGC::_gc_tracer = new SerialOldTracer();
675 SerialFullGC::_string_dedup_requests = new StringDedup::Requests();
676
677 // The Full GC operates on the entire heap so all objects should be subject
678 // to discovery, hence the _always_true_closure.
679 SerialFullGC::_ref_processor = new ReferenceProcessor(&_always_true_closure);
680 mark_and_push_closure.set_ref_discoverer(_ref_processor);
681 }
682
683 void SerialFullGC::invoke_at_safepoint(bool clear_all_softrefs) {
684 assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
685
686 SerialHeap* gch = SerialHeap::heap();
687 #ifdef ASSERT
688 if (gch->soft_ref_policy()->should_clear_all_soft_refs()) {
689 assert(clear_all_softrefs, "Policy should have been checked earlier");
690 }
691 #endif
692
693 gch->trace_heap_before_gc(_gc_tracer);
694
695 // Increment the invocation count
696 _total_invocations++;
697
698 // Capture used regions for old-gen to reestablish old-to-young invariant
699 // after full-gc.
700 gch->old_gen()->save_used_region();
701
702 allocate_stacks();
703
704 phase1_mark(clear_all_softrefs);
705
706 SlidingForwarding::begin();
707
708 Compacter compacter{gch};
709
710 {
711 // Now all live objects are marked, compute the new object addresses.
712 GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
713
714 compacter.phase2_calculate_new_addr();
715 }
716
717 // Don't add any more derived pointers during phase3
718 #if COMPILER2_OR_JVMCI
719 assert(DerivedPointerTable::is_active(), "Sanity");
720 DerivedPointerTable::set_active(false);
721 #endif
722
723 {
724 // Adjust the pointers to reflect the new locations
725 GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
726
727 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
728
729 NMethodToOopClosure code_closure(&adjust_pointer_closure, NMethodToOopClosure::FixRelocations);
730 gch->process_roots(SerialHeap::SO_AllCodeCache,
731 &adjust_pointer_closure,
732 &adjust_cld_closure,
733 &adjust_cld_closure,
734 &code_closure);
735
736 WeakProcessor::oops_do(&adjust_pointer_closure);
737
738 adjust_marks();
739 compacter.phase3_adjust_pointers();
740 }
741
742 {
743 // All pointers are now adjusted, move objects accordingly
744 GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
745
746 compacter.phase4_compact();
747 }
748
749 restore_marks();
750
751 // Set saved marks for allocation profiler (and other things? -- dld)
752 // (Should this be in general part?)
753 gch->save_marks();
754
755 SlidingForwarding::end();
756
757 deallocate_stacks();
758
759 SerialFullGC::_string_dedup_requests->flush();
760
761 bool is_young_gen_empty = (gch->young_gen()->used() == 0);
762 gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
763
764 gch->prune_scavengable_nmethods();
765
766 // Update heap occupancy information which is used as
767 // input to soft ref clearing policy at the next gc.
768 Universe::heap()->update_capacity_and_used_at_gc();
769
770 // Signal that we have completed a visit to all live objects.
771 Universe::heap()->record_whole_heap_examined_timestamp();
772
773 gch->trace_heap_after_gc(_gc_tracer);
774 }