170 // Used for BOT update
171 TenuredGeneration* _old_gen;
172
173 HeapWord* get_compaction_top(uint index) const {
174 return _spaces[index]._compaction_top;
175 }
176
177 HeapWord* get_first_dead(uint index) const {
178 return _spaces[index]._first_dead;
179 }
180
181 ContiguousSpace* get_space(uint index) const {
182 return _spaces[index]._space;
183 }
184
185 void record_first_dead(uint index, HeapWord* first_dead) {
186 assert(_spaces[index]._first_dead == nullptr, "should write only once");
187 _spaces[index]._first_dead = first_dead;
188 }
189
190 HeapWord* alloc(size_t words) {
191 while (true) {
192 if (words <= pointer_delta(_spaces[_index]._space->end(),
193 _spaces[_index]._compaction_top)) {
194 HeapWord* result = _spaces[_index]._compaction_top;
195 _spaces[_index]._compaction_top += words;
196 if (_index == 0) {
197 // old-gen requires BOT update
198 _old_gen->update_for_block(result, result + words);
199 }
200 return result;
201 }
202
203 // out-of-memory in this space
204 _index++;
205 assert(_index < max_num_spaces - 1, "the last space should not be used");
206 }
207 }
208
209 static void prefetch_read_scan(void* p) {
210 if (PrefetchScanIntervalInBytes >= 0) {
211 Prefetch::read(p, PrefetchScanIntervalInBytes);
212 }
213 }
214
215 static void prefetch_write_scan(void* p) {
216 if (PrefetchScanIntervalInBytes >= 0) {
217 Prefetch::write(p, PrefetchScanIntervalInBytes);
218 }
219 }
220
221 static void prefetch_write_copy(void* p) {
222 if (PrefetchCopyIntervalInBytes >= 0) {
223 Prefetch::write(p, PrefetchCopyIntervalInBytes);
224 }
225 }
226
227 static void forward_obj(oop obj, HeapWord* new_addr) {
228 prefetch_write_scan(obj);
229 if (cast_from_oop<HeapWord*>(obj) != new_addr) {
230 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
231 } else {
232 assert(obj->is_gc_marked(), "inv");
233 // This obj will stay in-place. Fix the markword.
234 obj->init_mark();
235 }
236 }
237
238 static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
239 for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
240 prefetch_read_scan(i_addr);
241 oop obj = cast_to_oop(i_addr);
242 if (obj->is_gc_marked()) {
243 return i_addr;
244 }
245 i_addr += obj->size();
246 }
247 return end;
248 };
249
250 static size_t relocate(HeapWord* addr) {
251 // Prefetch source and destination
252 prefetch_read_scan(addr);
253
254 oop obj = cast_to_oop(addr);
255 oop new_obj = FullGCForwarding::forwardee(obj);
256 HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
257 assert(addr != new_addr, "inv");
258 prefetch_write_copy(new_addr);
259
260 size_t obj_size = obj->size();
261 Copy::aligned_conjoint_words(addr, new_addr, obj_size);
262 new_obj->init_mark();
263
264 return obj_size;
265 }
266
267 public:
268 explicit Compacter(SerialHeap* heap) {
269 // In this order so that heap is compacted towards old-gen.
270 _spaces[0].init(heap->old_gen()->space());
271 _spaces[1].init(heap->young_gen()->eden());
272 _spaces[2].init(heap->young_gen()->from());
273
274 bool is_promotion_failed = !heap->young_gen()->to()->is_empty();
275 if (is_promotion_failed) {
276 _spaces[3].init(heap->young_gen()->to());
277 _num_spaces = 4;
278 } else {
279 _num_spaces = 3;
280 }
281 _index = 0;
282 _old_gen = heap->old_gen();
283 }
284
285 void phase2_calculate_new_addr() {
286 for (uint i = 0; i < _num_spaces; ++i) {
287 ContiguousSpace* space = get_space(i);
288 HeapWord* cur_addr = space->bottom();
289 HeapWord* top = space->top();
290
291 bool record_first_dead_done = false;
292
293 DeadSpacer dead_spacer(space);
294
295 while (cur_addr < top) {
296 oop obj = cast_to_oop(cur_addr);
297 size_t obj_size = obj->size();
298 if (obj->is_gc_marked()) {
299 HeapWord* new_addr = alloc(obj_size);
300 forward_obj(obj, new_addr);
301 cur_addr += obj_size;
302 } else {
303 // Skipping the current known-unmarked obj
304 HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
305 if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
306 // Register space for the filler obj
307 alloc(pointer_delta(next_live_addr, cur_addr));
308 } else {
309 if (!record_first_dead_done) {
310 record_first_dead(i, cur_addr);
311 record_first_dead_done = true;
312 }
313 *(HeapWord**)cur_addr = next_live_addr;
314 }
315 cur_addr = next_live_addr;
316 }
317 }
318
319 if (!record_first_dead_done) {
320 record_first_dead(i, top);
321 }
322 }
323 }
324
325 void phase3_adjust_pointers() {
326 for (uint i = 0; i < _num_spaces; ++i) {
327 ContiguousSpace* space = get_space(i);
580 if (_preserved_count_max != 0) {
581 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
582 young_gen->reset_scratch();
583 }
584
585 _preserved_overflow_stack_set.reclaim();
586 _marking_stack.clear();
587 _objarray_stack.clear(true);
588 }
589
590 void SerialFullGC::mark_object(oop obj) {
591 if (StringDedup::is_enabled() &&
592 java_lang_String::is_instance(obj) &&
593 SerialStringDedup::is_candidate_from_mark(obj)) {
594 _string_dedup_requests->add(obj);
595 }
596
597 // some marks may contain information we need to preserve so we store them away
598 // and overwrite the mark. We'll restore it at the end of serial full GC.
599 markWord mark = obj->mark();
600 obj->set_mark(obj->prototype_mark().set_marked());
601
602 ContinuationGCSupport::transform_stack_chunk(obj);
603
604 if (obj->mark_must_be_preserved(mark)) {
605 preserve_mark(obj, mark);
606 }
607 }
608
609 template <class T> void SerialFullGC::mark_and_push(T* p) {
610 T heap_oop = RawAccess<>::oop_load(p);
611 if (!CompressedOops::is_null(heap_oop)) {
612 oop obj = CompressedOops::decode_not_null(heap_oop);
613 if (!obj->mark().is_marked()) {
614 mark_object(obj);
615 _marking_stack.push(obj);
616 }
617 }
618 }
619
620 template <typename T>
693 gch->trace_heap_before_gc(_gc_tracer);
694
695 // Capture used regions for old-gen to reestablish old-to-young invariant
696 // after full-gc.
697 gch->old_gen()->save_used_region();
698
699 allocate_stacks();
700
701 // Usually, all class unloading work occurs at the end of phase 1, but Serial
702 // full-gc accesses dead-objs' klass to find out the start of next live-obj
703 // during phase 2. This requires klasses of dead-objs to be kept loaded.
704 // Therefore, we declare ClassUnloadingContext at the same level as
705 // full-gc phases, and purge dead classes (invoking
706 // ClassLoaderDataGraph::purge) after all phases of full-gc.
707 ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
708 false /* unregister_nmethods_during_purge */,
709 false /* lock_nmethod_free_separately */);
710
711 phase1_mark(clear_all_softrefs);
712
713 Compacter compacter{gch};
714
715 {
716 // Now all live objects are marked, compute the new object addresses.
717 GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
718
719 compacter.phase2_calculate_new_addr();
720 }
721
722 // Don't add any more derived pointers during phase3
723 #ifdef COMPILER2
724 assert(DerivedPointerTable::is_active(), "Sanity");
725 DerivedPointerTable::set_active(false);
726 #endif // COMPILER2
727
728 {
729 // Adjust the pointers to reflect the new locations
730 GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
731
732 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
750 adjust_marks();
751 compacter.phase3_adjust_pointers();
752 }
753
754 {
755 // All pointers are now adjusted, move objects accordingly
756 GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
757
758 compacter.phase4_compact();
759 }
760
761 // Delete metaspaces for unloaded class loaders and clean up CLDG.
762 ClassLoaderDataGraph::purge(true /* at_safepoint */);
763 DEBUG_ONLY(MetaspaceUtils::verify();)
764
765 // Need to clear claim bits for the next full-gc (specifically phase 1 and 3).
766 ClassLoaderDataGraph::clear_claimed_marks();
767
768 restore_marks();
769
770 deallocate_stacks();
771
772 SerialFullGC::_string_dedup_requests->flush();
773
774 bool is_young_gen_empty = (gch->young_gen()->used() == 0);
775 gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
776
777 gch->prune_scavengable_nmethods();
778
779 // Update heap occupancy information which is used as
780 // input to soft ref clearing policy at the next gc.
781 Universe::heap()->update_capacity_and_used_at_gc();
782
783 // Signal that we have completed a visit to all live objects.
784 Universe::heap()->record_whole_heap_examined_timestamp();
785
786 gch->trace_heap_after_gc(_gc_tracer);
787 }
|
170 // Used for BOT update
171 TenuredGeneration* _old_gen;
172
173 HeapWord* get_compaction_top(uint index) const {
174 return _spaces[index]._compaction_top;
175 }
176
177 HeapWord* get_first_dead(uint index) const {
178 return _spaces[index]._first_dead;
179 }
180
181 ContiguousSpace* get_space(uint index) const {
182 return _spaces[index]._space;
183 }
184
185 void record_first_dead(uint index, HeapWord* first_dead) {
186 assert(_spaces[index]._first_dead == nullptr, "should write only once");
187 _spaces[index]._first_dead = first_dead;
188 }
189
190 HeapWord* alloc(size_t old_size, size_t new_size, HeapWord* old_obj) {
191 size_t words = (old_obj == _spaces[_index]._compaction_top) ? old_size : new_size;
192 while (true) {
193 if (words <= pointer_delta(_spaces[_index]._space->end(),
194 _spaces[_index]._compaction_top)) {
195 HeapWord* result = _spaces[_index]._compaction_top;
196 _spaces[_index]._compaction_top += words;
197 if (_index == 0) {
198 // old-gen requires BOT update
199 _old_gen->update_for_block(result, result + words);
200 }
201 return result;
202 }
203
204 // out-of-memory in this space
205 _index++;
206 assert(_index < max_num_spaces - 1, "the last space should not be used");
207 words = (old_obj == _spaces[_index]._compaction_top) ? old_size : new_size;
208 }
209 }
210
211 static void prefetch_read_scan(void* p) {
212 if (PrefetchScanIntervalInBytes >= 0) {
213 Prefetch::read(p, PrefetchScanIntervalInBytes);
214 }
215 }
216
217 static void prefetch_write_scan(void* p) {
218 if (PrefetchScanIntervalInBytes >= 0) {
219 Prefetch::write(p, PrefetchScanIntervalInBytes);
220 }
221 }
222
223 static void prefetch_write_copy(void* p) {
224 if (PrefetchCopyIntervalInBytes >= 0) {
225 Prefetch::write(p, PrefetchCopyIntervalInBytes);
226 }
227 }
228
229 static void forward_obj(oop obj, HeapWord* new_addr, bool after_first_dead) {
230 prefetch_write_scan(obj);
231 if (cast_from_oop<HeapWord*>(obj) != new_addr) {
232 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
233 } else {
234 assert(obj->is_gc_marked(), "inv");
235 if (!after_first_dead) {
236 // This obj will stay in-place and we'll not see it during relocation.
237 // Fix the markword.
238 obj->reinit_mark();
239 } else {
240 FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
241 }
242 }
243 }
244
245 static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
246 for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
247 prefetch_read_scan(i_addr);
248 oop obj = cast_to_oop(i_addr);
249 if (obj->is_gc_marked()) {
250 return i_addr;
251 }
252 i_addr += obj->size();
253 }
254 return end;
255 };
256
257 static size_t relocate(HeapWord* addr) {
258 // Prefetch source and destination
259 prefetch_read_scan(addr);
260
261 oop obj = cast_to_oop(addr);
262 oop new_obj = FullGCForwarding::forwardee(obj);
263 HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
264
265 size_t obj_size = obj->size();
266 if (addr != new_addr) {
267 prefetch_write_copy(new_addr);
268 Copy::aligned_conjoint_words(addr, new_addr, obj_size);
269 }
270 new_obj->reinit_mark();
271 if (addr != new_addr) {
272 new_obj->initialize_hash_if_necessary(obj);
273 }
274
275 return obj_size;
276 }
277
278 public:
279 explicit Compacter(SerialHeap* heap) {
280 // In this order so that heap is compacted towards old-gen.
281 _spaces[0].init(heap->old_gen()->space());
282 _spaces[1].init(heap->young_gen()->eden());
283 _spaces[2].init(heap->young_gen()->from());
284
285 bool is_promotion_failed = !heap->young_gen()->to()->is_empty();
286 if (is_promotion_failed) {
287 _spaces[3].init(heap->young_gen()->to());
288 _num_spaces = 4;
289 } else {
290 _num_spaces = 3;
291 }
292 _index = 0;
293 _old_gen = heap->old_gen();
294 }
295
296 void phase2_calculate_new_addr() {
297 for (uint i = 0; i < _num_spaces; ++i) {
298 ContiguousSpace* space = get_space(i);
299 HeapWord* cur_addr = space->bottom();
300 HeapWord* top = space->top();
301
302 bool record_first_dead_done = false;
303
304 DeadSpacer dead_spacer(space);
305
306 while (cur_addr < top) {
307 oop obj = cast_to_oop(cur_addr);
308 size_t obj_size = obj->size();
309 size_t new_size = obj->copy_size(obj_size, obj->mark());
310 if (obj->is_gc_marked()) {
311 HeapWord* new_addr = alloc(obj_size, new_size, cur_addr);
312 forward_obj(obj, new_addr, record_first_dead_done);
313 assert(obj->size() == obj_size, "size must not change after forwarding");
314 cur_addr += obj_size;
315 } else {
316 // Skipping the current known-unmarked obj
317 HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
318 if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
319 // Register space for the filler obj
320 size_t size = pointer_delta(next_live_addr, cur_addr);
321 alloc(size, size, cur_addr);
322 } else {
323 if (!record_first_dead_done) {
324 record_first_dead(i, cur_addr);
325 record_first_dead_done = true;
326 }
327 *(HeapWord**)cur_addr = next_live_addr;
328 }
329 cur_addr = next_live_addr;
330 }
331 }
332
333 if (!record_first_dead_done) {
334 record_first_dead(i, top);
335 }
336 }
337 }
338
339 void phase3_adjust_pointers() {
340 for (uint i = 0; i < _num_spaces; ++i) {
341 ContiguousSpace* space = get_space(i);
594 if (_preserved_count_max != 0) {
595 DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
596 young_gen->reset_scratch();
597 }
598
599 _preserved_overflow_stack_set.reclaim();
600 _marking_stack.clear();
601 _objarray_stack.clear(true);
602 }
603
604 void SerialFullGC::mark_object(oop obj) {
605 if (StringDedup::is_enabled() &&
606 java_lang_String::is_instance(obj) &&
607 SerialStringDedup::is_candidate_from_mark(obj)) {
608 _string_dedup_requests->add(obj);
609 }
610
611 // some marks may contain information we need to preserve so we store them away
612 // and overwrite the mark. We'll restore it at the end of serial full GC.
613 markWord mark = obj->mark();
614 obj->set_mark(mark.set_marked());
615
616 ContinuationGCSupport::transform_stack_chunk(obj);
617
618 if (obj->mark_must_be_preserved(mark)) {
619 preserve_mark(obj, mark);
620 }
621 }
622
623 template <class T> void SerialFullGC::mark_and_push(T* p) {
624 T heap_oop = RawAccess<>::oop_load(p);
625 if (!CompressedOops::is_null(heap_oop)) {
626 oop obj = CompressedOops::decode_not_null(heap_oop);
627 if (!obj->mark().is_marked()) {
628 mark_object(obj);
629 _marking_stack.push(obj);
630 }
631 }
632 }
633
634 template <typename T>
707 gch->trace_heap_before_gc(_gc_tracer);
708
709 // Capture used regions for old-gen to reestablish old-to-young invariant
710 // after full-gc.
711 gch->old_gen()->save_used_region();
712
713 allocate_stacks();
714
715 // Usually, all class unloading work occurs at the end of phase 1, but Serial
716 // full-gc accesses dead-objs' klass to find out the start of next live-obj
717 // during phase 2. This requires klasses of dead-objs to be kept loaded.
718 // Therefore, we declare ClassUnloadingContext at the same level as
719 // full-gc phases, and purge dead classes (invoking
720 // ClassLoaderDataGraph::purge) after all phases of full-gc.
721 ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
722 false /* unregister_nmethods_during_purge */,
723 false /* lock_nmethod_free_separately */);
724
725 phase1_mark(clear_all_softrefs);
726
727 FullGCForwarding::begin();
728
729 Compacter compacter{gch};
730
731 {
732 // Now all live objects are marked, compute the new object addresses.
733 GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
734
735 compacter.phase2_calculate_new_addr();
736 }
737
738 // Don't add any more derived pointers during phase3
739 #ifdef COMPILER2
740 assert(DerivedPointerTable::is_active(), "Sanity");
741 DerivedPointerTable::set_active(false);
742 #endif // COMPILER2
743
744 {
745 // Adjust the pointers to reflect the new locations
746 GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
747
748 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
766 adjust_marks();
767 compacter.phase3_adjust_pointers();
768 }
769
770 {
771 // All pointers are now adjusted, move objects accordingly
772 GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
773
774 compacter.phase4_compact();
775 }
776
777 // Delete metaspaces for unloaded class loaders and clean up CLDG.
778 ClassLoaderDataGraph::purge(true /* at_safepoint */);
779 DEBUG_ONLY(MetaspaceUtils::verify();)
780
781 // Need to clear claim bits for the next full-gc (specifically phase 1 and 3).
782 ClassLoaderDataGraph::clear_claimed_marks();
783
784 restore_marks();
785
786 FullGCForwarding::end();
787
788 deallocate_stacks();
789
790 SerialFullGC::_string_dedup_requests->flush();
791
792 bool is_young_gen_empty = (gch->young_gen()->used() == 0);
793 gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
794
795 gch->prune_scavengable_nmethods();
796
797 // Update heap occupancy information which is used as
798 // input to soft ref clearing policy at the next gc.
799 Universe::heap()->update_capacity_and_used_at_gc();
800
801 // Signal that we have completed a visit to all live objects.
802 Universe::heap()->record_whole_heap_examined_timestamp();
803
804 gch->trace_heap_after_gc(_gc_tracer);
805 }
|