1 /*
2 * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25
26 #include "compiler/oopMap.hpp"
27 #include "cppstdlib/new.hpp"
28 #include "gc/g1/g1CardSetMemory.hpp"
29 #include "gc/g1/g1CardTableEntryClosure.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectionSetCandidates.inline.hpp"
32 #include "gc/g1/g1CollectorState.inline.hpp"
33 #include "gc/g1/g1ConcurrentMark.inline.hpp"
34 #include "gc/g1/g1EvacFailureRegions.inline.hpp"
35 #include "gc/g1/g1EvacInfo.hpp"
36 #include "gc/g1/g1EvacStats.inline.hpp"
37 #include "gc/g1/g1HeapRegion.inline.hpp"
38 #include "gc/g1/g1HeapRegionPrinter.hpp"
39 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
40 #include "gc/g1/g1OopClosures.inline.hpp"
41 #include "gc/g1/g1ParScanThreadState.hpp"
42 #include "gc/g1/g1RemSet.hpp"
43 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
44 #include "gc/shared/bufferNode.hpp"
45 #include "gc/shared/partialArrayState.hpp"
46 #include "jfr/jfrEvents.hpp"
47 #include "oops/access.inline.hpp"
48 #include "oops/compressedOops.inline.hpp"
49 #include "oops/oop.inline.hpp"
50 #include "runtime/atomic.hpp"
51 #include "runtime/prefetch.inline.hpp"
52 #include "runtime/threads.hpp"
53 #include "runtime/threadSMR.hpp"
54 #include "utilities/bitMap.inline.hpp"
55 #include "utilities/ticks.hpp"
56
57 class G1PostEvacuateCollectionSetCleanupTask1::FlushPssTask : public G1AbstractSubTask {
58 G1ParScanThreadStateSet* _per_thread_states;
59
60 public:
61 FlushPssTask(G1ParScanThreadStateSet* per_thread_states) :
62 G1AbstractSubTask(G1GCPhaseTimes::FlushPSS),
63 _per_thread_states(per_thread_states) { }
64
65 double worker_cost() const override { return 1.0; }
66
67 void do_work(uint worker_id) override { _per_thread_states->flush_stats(); }
68 };
69
70 class G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask : public G1AbstractSubTask {
71 bool _evacuation_failed;
72 bool _allocation_failed;
73
74 public:
75 RecalculateUsedTask(bool evacuation_failed, bool allocation_failed) :
76 G1AbstractSubTask(G1GCPhaseTimes::RecalculateUsed),
77 _evacuation_failed(evacuation_failed),
78 _allocation_failed(allocation_failed) { }
79
80 double worker_cost() const override {
81 // If there is no evacuation failure, the work to perform is minimal.
82 return _evacuation_failed ? 1.0 : AlmostNoWork;
83 }
84
85 void do_work(uint worker_id) override {
86 G1CollectedHeap::heap()->update_used_after_gc(_evacuation_failed);
87 if (_allocation_failed) {
88 // Reset the G1GCAllocationFailureALot counters and flags
89 G1CollectedHeap::heap()->allocation_failure_injector()->reset();
90 }
91 }
92 };
93
94 class G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask : public G1AbstractSubTask {
95 public:
96 SampleCollectionSetCandidatesTask() : G1AbstractSubTask(G1GCPhaseTimes::SampleCollectionSetCandidates) { }
97
98 static bool should_execute() {
99 return G1CollectedHeap::heap()->should_sample_collection_set_candidates();
100 }
101
102 double worker_cost() const override {
103 return should_execute() ? 1.0 : AlmostNoWork;
104 }
105
106 void do_work(uint worker_id) override {
107 G1CollectedHeap* g1h = G1CollectedHeap::heap();
108
109 G1MonotonicArenaMemoryStats _total;
110 G1CollectionSetCandidates* candidates = g1h->collection_set()->candidates();
111 for (G1CSetCandidateGroup* gr : candidates->from_marking_groups()) {
112 _total.add(gr->card_set_memory_stats());
113 }
114
115 for (G1CSetCandidateGroup* gr : candidates->retained_groups()) {
116 _total.add(gr->card_set_memory_stats());
117 }
118 g1h->set_collection_set_candidates_stats(_total);
119 }
120 };
121
122 class G1PostEvacuateCollectionSetCleanupTask1::UpdateCodeRootsTask
123 : public G1AbstractSubTask
124 {
125 class ProcessRegionClosure : public G1HeapRegionClosure {
126 G1ParScanThreadStateSet* _psss;
127
128 public:
129 ProcessRegionClosure(G1ParScanThreadStateSet* psss) : _psss(psss) { }
130
131 bool do_heap_region(G1HeapRegion* r) override {
132 uint index = r->hrm_index();
133
134 size_t num_nmethods = 0;
135 for (uint i = 0; i < _psss->num_workers(); i++) {
136 G1ParScanThreadState* pss = _psss->state_for_worker(i);
137 num_nmethods += pss->num_nmethods(index);
138 }
139 if (num_nmethods != 0) {
140 // Notify the code root sets that we are going to add code roots.
141 r->rem_set()->prepare_for_adding_code_roots(num_nmethods);
142
143 // Add roots.
144 for (uint i = 0; i < _psss->num_workers(); i++) {
145 G1ParScanThreadState* pss = _psss->state_for_worker(i);
146 pss->iterate_nmethods(index, [&] (nmethod* nm) { r->add_code_root(nm); });
147 }
148 }
149 return false;
150 }
151 };
152
153 G1ParScanThreadStateSet* _psss;
154 G1HeapRegionClaimer _claimer;
155
156 public:
157 UpdateCodeRootsTask(G1ParScanThreadStateSet* per_thread_states)
158 : G1AbstractSubTask(G1GCPhaseTimes::UpdateCodeRoots), _psss(per_thread_states), _claimer(0) { }
159
160 double worker_cost() const override {
161 return _psss->num_nmethod_regions_to_add();
162 }
163
164 void set_max_workers(uint max_workers) override {
165 _claimer.set_n_workers(max_workers);
166 }
167
168 void do_work(uint worker_id) override {
169 ProcessRegionClosure cl(_psss);
170 _psss->par_iterate_nmethod_regions_to_add(&cl, &_claimer, worker_id);
171 }
172 };
173
174 class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : public G1AbstractSubTask {
175 G1CollectedHeap* _g1h;
176 G1ConcurrentMark* _cm;
177
178 G1EvacFailureRegions* _evac_failure_regions;
179 CHeapBitMap _chunk_bitmap;
180
181 uint _num_chunks_per_region;
182 uint _num_evac_fail_regions;
183 size_t _chunk_size;
184
185 class PhaseTimesStat {
186 static constexpr G1GCPhaseTimes::GCParPhases phase_name =
187 G1GCPhaseTimes::RemoveSelfForwards;
188
189 G1GCPhaseTimes* _phase_times;
190 uint _worker_id;
191 Ticks _start;
192
193 public:
194 PhaseTimesStat(G1GCPhaseTimes* phase_times, uint worker_id) :
195 _phase_times(phase_times),
196 _worker_id(worker_id),
197 _start(Ticks::now()) { }
198
199 ~PhaseTimesStat() {
200 _phase_times->record_or_add_time_secs(phase_name,
201 _worker_id,
202 (Ticks::now() - _start).seconds());
203 }
204
205 void register_empty_chunk() {
206 _phase_times->record_or_add_thread_work_item(phase_name,
207 _worker_id,
208 1,
209 G1GCPhaseTimes::RemoveSelfForwardEmptyChunksNum);
210 }
211
212 void register_nonempty_chunk() {
213 _phase_times->record_or_add_thread_work_item(phase_name,
214 _worker_id,
215 1,
216 G1GCPhaseTimes::RemoveSelfForwardChunksNum);
217 }
218
219 void register_objects_count_and_size(size_t num_marked_obj, size_t marked_words) {
220 _phase_times->record_or_add_thread_work_item(phase_name,
221 _worker_id,
222 num_marked_obj,
223 G1GCPhaseTimes::RemoveSelfForwardObjectsNum);
224
225 size_t marked_bytes = marked_words * HeapWordSize;
226 _phase_times->record_or_add_thread_work_item(phase_name,
227 _worker_id,
228 marked_bytes,
229 G1GCPhaseTimes::RemoveSelfForwardObjectsBytes);
230 }
231 };
232
233 // Fill the memory area from start to end with filler objects, and update the BOT
234 // accordingly. Since we clear and use the bitmap for marking objects that failed
235 // evacuation, there is no other work to be done there.
236 static size_t zap_dead_objects(G1HeapRegion* hr, HeapWord* start, HeapWord* end) {
237 assert(start <= end, "precondition");
238 if (start == end) {
239 return 0;
240 }
241
242 hr->fill_range_with_dead_objects(start, end);
243 return pointer_delta(end, start);
244 }
245
246 static void update_garbage_words_in_hr(G1HeapRegion* hr, size_t garbage_words) {
247 if (garbage_words != 0) {
248 hr->note_self_forward_chunk_done(garbage_words * HeapWordSize);
249 }
250 }
251
252 static void prefetch_obj(HeapWord* obj_addr) {
253 Prefetch::write(obj_addr, PrefetchScanIntervalInBytes);
254 }
255
256 bool claim_chunk(uint chunk_idx) {
257 return _chunk_bitmap.par_set_bit(chunk_idx);
258 }
259
260 void process_chunk(uint worker_id, uint chunk_idx) {
261 PhaseTimesStat stat(_g1h->phase_times(), worker_id);
262
263 G1CMBitMap* bitmap = _cm->mark_bitmap();
264 const uint region_idx = _evac_failure_regions->get_region_idx(chunk_idx / _num_chunks_per_region);
265 G1HeapRegion* hr = _g1h->region_at(region_idx);
266
267 HeapWord* hr_bottom = hr->bottom();
268 HeapWord* hr_top = hr->top();
269 HeapWord* chunk_start = hr_bottom + (chunk_idx % _num_chunks_per_region) * _chunk_size;
270
271 assert(chunk_start < hr->end(), "inv");
272 if (chunk_start >= hr_top) {
273 return;
274 }
275
276 HeapWord* chunk_end = MIN2(chunk_start + _chunk_size, hr_top);
277 HeapWord* first_marked_addr = bitmap->get_next_marked_addr(chunk_start, hr_top);
278
279 size_t garbage_words = 0;
280
281 if (chunk_start == hr_bottom) {
282 // This is the bottom-most chunk in this region; zap [bottom, first_marked_addr).
283 garbage_words += zap_dead_objects(hr, hr_bottom, first_marked_addr);
284 }
285
286 if (first_marked_addr >= chunk_end) {
287 stat.register_empty_chunk();
288 update_garbage_words_in_hr(hr, garbage_words);
289 return;
290 }
291
292 stat.register_nonempty_chunk();
293
294 size_t num_marked_objs = 0;
295 size_t marked_words = 0;
296
297 HeapWord* obj_addr = first_marked_addr;
298 assert(chunk_start <= obj_addr && obj_addr < chunk_end,
299 "object " PTR_FORMAT " must be within chunk [" PTR_FORMAT ", " PTR_FORMAT "[",
300 p2i(obj_addr), p2i(chunk_start), p2i(chunk_end));
301 do {
302 assert(bitmap->is_marked(obj_addr), "inv");
303 prefetch_obj(obj_addr);
304
305 oop obj = cast_to_oop(obj_addr);
306 const size_t obj_size = obj->size();
307 HeapWord* const obj_end_addr = obj_addr + obj_size;
308
309 {
310 // Process marked object.
311 assert(obj->is_self_forwarded(), "must be self-forwarded");
312 obj->unset_self_forwarded();
313 hr->update_bot_for_block(obj_addr, obj_end_addr);
314
315 // Statistics
316 num_marked_objs++;
317 marked_words += obj_size;
318 }
319
320 assert(obj_end_addr <= hr_top, "inv");
321 // Use hr_top as the limit so that we zap dead ranges up to the next
322 // marked obj or hr_top.
323 HeapWord* next_marked_obj_addr = bitmap->get_next_marked_addr(obj_end_addr, hr_top);
324 garbage_words += zap_dead_objects(hr, obj_end_addr, next_marked_obj_addr);
325 obj_addr = next_marked_obj_addr;
326 } while (obj_addr < chunk_end);
327
328 assert(marked_words > 0 && num_marked_objs > 0, "inv");
329
330 stat.register_objects_count_and_size(num_marked_objs, marked_words);
331
332 update_garbage_words_in_hr(hr, garbage_words);
333 }
334
335 public:
336 RestoreEvacFailureRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
337 G1AbstractSubTask(G1GCPhaseTimes::RestoreEvacuationFailedRegions),
338 _g1h(G1CollectedHeap::heap()),
339 _cm(_g1h->concurrent_mark()),
340 _evac_failure_regions(evac_failure_regions),
341 _chunk_bitmap(mtGC) {
342
343 _num_evac_fail_regions = _evac_failure_regions->num_regions_evac_failed();
344 _num_chunks_per_region = G1CollectedHeap::get_chunks_per_region_for_scan();
345
346 _chunk_size = static_cast<uint>(G1HeapRegion::GrainWords / _num_chunks_per_region);
347
348 log_debug(gc, ergo)("Initializing removing self forwards with %u chunks per region",
349 _num_chunks_per_region);
350
351 _chunk_bitmap.resize(_num_chunks_per_region * _num_evac_fail_regions);
352 }
353
354 double worker_cost() const override {
355 assert(_evac_failure_regions->has_regions_evac_failed(), "Should not call this if there were no evacuation failures");
356
357 double workers_per_region = (double)G1CollectedHeap::get_chunks_per_region_for_scan() / G1RestoreRetainedRegionChunksPerWorker;
358 return workers_per_region * _evac_failure_regions->num_regions_evac_failed();
359 }
360
361 void do_work(uint worker_id) override {
362 const uint total_workers = G1CollectedHeap::heap()->workers()->active_workers();
363 const uint total_chunks = _num_chunks_per_region * _num_evac_fail_regions;
364 const uint start_chunk_idx = worker_id * total_chunks / total_workers;
365
366 for (uint i = 0; i < total_chunks; i++) {
367 const uint chunk_idx = (start_chunk_idx + i) % total_chunks;
368 if (claim_chunk(chunk_idx)) {
369 process_chunk(worker_id, chunk_idx);
370 }
371 }
372 }
373 };
374
375 G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
376 G1EvacFailureRegions* evac_failure_regions) :
377 G1BatchedTask("Post Evacuate Cleanup 1", G1CollectedHeap::heap()->phase_times())
378 {
379 bool evac_failed = evac_failure_regions->has_regions_evac_failed();
380 bool alloc_failed = evac_failure_regions->has_regions_alloc_failed();
381
382 add_serial_task(new FlushPssTask(per_thread_states));
383 add_serial_task(new RecalculateUsedTask(evac_failed, alloc_failed));
384 if (SampleCollectionSetCandidatesTask::should_execute()) {
385 add_serial_task(new SampleCollectionSetCandidatesTask());
386 }
387 add_parallel_task(new UpdateCodeRootsTask(per_thread_states));
388
389 add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
390 if (evac_failed) {
391 add_parallel_task(new RestoreEvacFailureRegionsTask(evac_failure_regions));
392 }
393 }
394
395 class G1FreeHumongousRegionClosure : public G1HeapRegionIndexClosure {
396 uint _humongous_objects_reclaimed;
397 uint _humongous_regions_reclaimed;
398 size_t _freed_bytes;
399 G1CollectedHeap* _g1h;
400
401 // Returns whether the given humongous object defined by the start region index
402 // is reclaimable.
403 //
404 // At this point in the garbage collection, checking whether the humongous object
405 // is still a candidate is sufficient because:
406 //
407 // - if it has not been a candidate at the start of collection, it will never
408 // changed to be a candidate during the gc (and live).
409 // - any found outstanding (i.e. in its remembered set, or from the collection
410 // set) references will set the candidate state to false.
411 // - there can be no references from within humongous starts regions referencing
412 // the object because we never allocate other objects into them.
413 // (I.e. there can be no intra-region references within humongous objects)
414 //
415 // It is not required to check whether the object has been found dead by marking
416 // or not, in fact it would prevent reclamation within a concurrent cycle, as
417 // all objects allocated during that time are considered live.
418 // SATB marking is even more conservative than the remembered set.
419 // So if at this point in the collection we did not find a reference during gc
420 // (or it had enough references to not be a candidate, having many remembered
421 // set entries), nobody has a reference to it.
422 //
423 // Since remembered sets are only ever updated by concurrent refinement threads
424 // at mutator time, the remembered sets do not need to be checked again.
425 //
426 // Other implementation considerations:
427 // - never consider non-typeArrays during marking as there is a considerable cost
428 // for maintaining the SATB invariant.
429 bool is_reclaimable(uint region_idx) const {
430 return G1CollectedHeap::heap()->is_humongous_reclaim_candidate(region_idx);
431 }
432
433 public:
434 G1FreeHumongousRegionClosure() :
435 _humongous_objects_reclaimed(0),
436 _humongous_regions_reclaimed(0),
437 _freed_bytes(0),
438 _g1h(G1CollectedHeap::heap())
439 {}
440
441 bool do_heap_region_index(uint region_index) override {
442 if (!is_reclaimable(region_index)) {
443 return false;
444 }
445
446 G1HeapRegion* r = _g1h->region_at(region_index);
447
448 oop obj = cast_to_oop(r->bottom());
449 {
450 ResourceMark rm;
451 bool mark_in_progress = _g1h->collector_state()->is_in_marking();
452 bool allocated_after_mark_start = false;
453 if (mark_in_progress) {
454 // top_at_mark_start() will assert if we are not in marking, so check first.
455 allocated_after_mark_start = r->bottom() == _g1h->concurrent_mark()->top_at_mark_start(r);
456 }
457
458 guarantee(obj->is_typeArray() || (allocated_after_mark_start || !mark_in_progress),
459 "Only eagerly reclaiming primitive arrays is supported, other humongous objects only if allocated after mark start, but the object "
460 PTR_FORMAT " (%s) is not (mark %d allocated after mark: %d).",
461 p2i(r->bottom()), obj->klass()->name()->as_C_string(), mark_in_progress, allocated_after_mark_start);
462 }
463 log_debug(gc, humongous)("Reclaimed humongous region %u (object size %zu @ " PTR_FORMAT ")",
464 region_index,
465 obj->size() * HeapWordSize,
466 p2i(r->bottom())
467 );
468
469 G1ConcurrentMark* const cm = _g1h->concurrent_mark();
470 cm->humongous_object_eagerly_reclaimed(r);
471 assert(!cm->is_marked_in_bitmap(obj),
472 "Eagerly reclaimed humongous region %u should not be marked at all but is in bitmap %s",
473 region_index,
474 BOOL_TO_STR(cm->is_marked_in_bitmap(obj)));
475 _humongous_objects_reclaimed++;
476
477 auto free_humongous_region = [&] (G1HeapRegion* r) {
478 _freed_bytes += r->used();
479 r->set_containing_set(nullptr);
480 _humongous_regions_reclaimed++;
481 G1HeapRegionPrinter::eager_reclaim(r);
482 // Humongous non-typeArrays may have dirty card tables. Need to be cleared. Do it
483 // for all types just in case.
484 r->clear_both_card_tables();
485 _g1h->free_humongous_region(r, nullptr);
486 };
487
488 _g1h->humongous_obj_regions_iterate(r, free_humongous_region);
489
490 return false;
491 }
492
493 uint humongous_objects_reclaimed() {
494 return _humongous_objects_reclaimed;
495 }
496
497 uint humongous_regions_reclaimed() {
498 return _humongous_regions_reclaimed;
499 }
500
501 size_t bytes_freed() const {
502 return _freed_bytes;
503 }
504 };
505
506 #ifdef COMPILER2
507 class G1PostEvacuateCollectionSetCleanupTask2::UpdateDerivedPointersTask : public G1AbstractSubTask {
508 public:
509 UpdateDerivedPointersTask() : G1AbstractSubTask(G1GCPhaseTimes::UpdateDerivedPointers) { }
510
511 double worker_cost() const override { return 1.0; }
512 void do_work(uint worker_id) override { DerivedPointerTable::update_pointers(); }
513 };
514 #endif // COMPILER2
515
516 class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask : public G1AbstractSubTask {
517 uint _humongous_regions_reclaimed;
518 size_t _bytes_freed;
519
520 public:
521 EagerlyReclaimHumongousObjectsTask() :
522 G1AbstractSubTask(G1GCPhaseTimes::EagerlyReclaimHumongousObjects),
523 _humongous_regions_reclaimed(0),
524 _bytes_freed(0) { }
525
526 virtual ~EagerlyReclaimHumongousObjectsTask() {
527 G1CollectedHeap* g1h = G1CollectedHeap::heap();
528
529 g1h->remove_from_old_gen_sets(0, _humongous_regions_reclaimed);
530 g1h->decrement_summary_bytes(_bytes_freed);
531 }
532
533 double worker_cost() const override { return 1.0; }
534 void do_work(uint worker_id) override {
535 G1CollectedHeap* g1h = G1CollectedHeap::heap();
536
537 G1FreeHumongousRegionClosure cl;
538 g1h->heap_region_iterate(&cl);
539
540 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumTotal, g1h->num_humongous_objects());
541 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumCandidates, g1h->num_humongous_reclaim_candidates());
542 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumReclaimed, cl.humongous_objects_reclaimed());
543
544 _humongous_regions_reclaimed = cl.humongous_regions_reclaimed();
545 _bytes_freed = cl.bytes_freed();
546 }
547 };
548
549 class G1PostEvacuateCollectionSetCleanupTask2::ProcessEvacuationFailedRegionsTask : public G1AbstractSubTask {
550 G1EvacFailureRegions* _evac_failure_regions;
551 G1HeapRegionClaimer _claimer;
552
553 class ProcessEvacuationFailedRegionsClosure : public G1HeapRegionClosure {
554 public:
555
556 bool do_heap_region(G1HeapRegion* r) override {
557 G1CollectedHeap* g1h = G1CollectedHeap::heap();
558 G1ConcurrentMark* cm = g1h->concurrent_mark();
559
560 // Retained regions are root regions for marking, so we must clear their mark data
561 // (tams, bitmap, ...). Outside of Concurrent Start GC we must always clear the mark data
562 // for the next GC.
563 bool clear_mark_data = !g1h->collector_state()->is_in_concurrent_start_gc() ||
564 g1h->policy()->should_retain_evac_failed_region(r);
565
566 if (clear_mark_data) {
567 g1h->clear_bitmap_for_region(r);
568 // Must be because this is a region that should not have been selected to
569 // be marked through.
570 cm->assert_top_at_mark_start_is_bottom(r);
571 } else {
572 // This evacuation failed region is going to be marked through. Update mark data.
573 // Since we have some marked live data information, pass that too.
574 cm->assert_statistics_clear(r);
575 cm->notify_new_region(r, r->live_bytes());
576 }
577 return false;
578 }
579 };
580
581 public:
582 ProcessEvacuationFailedRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
583 G1AbstractSubTask(G1GCPhaseTimes::ProcessEvacuationFailedRegions),
584 _evac_failure_regions(evac_failure_regions),
585 _claimer(0) {
586 }
587
588 void set_max_workers(uint max_workers) override {
589 _claimer.set_n_workers(max_workers);
590 }
591
592 double worker_cost() const override {
593 return _evac_failure_regions->num_regions_evac_failed();
594 }
595
596 void do_work(uint worker_id) override {
597 ProcessEvacuationFailedRegionsClosure cl;
598 _evac_failure_regions->par_iterate(&cl, &_claimer, worker_id);
599 }
600 };
601
602 // Helper class to keep statistics for the collection set freeing
603 class FreeCSetStats {
604 size_t _before_used_bytes; // Usage in regions successfully evacuate
605 size_t _after_used_bytes; // Usage in regions failing evacuation
606 size_t _bytes_allocated_in_old_since_last_pause; // Size of young regions turned into old
607 size_t _failure_used_words; // Live size in failed regions
608 size_t _failure_waste_words; // Wasted size in failed regions
609 uint _regions_freed; // Number of regions freed
610
611 public:
612 FreeCSetStats() :
613 _before_used_bytes(0),
614 _after_used_bytes(0),
615 _bytes_allocated_in_old_since_last_pause(0),
616 _failure_used_words(0),
617 _failure_waste_words(0),
618 _regions_freed(0) { }
619
620 void merge_stats(FreeCSetStats* other) {
621 assert(other != nullptr, "invariant");
622 _before_used_bytes += other->_before_used_bytes;
623 _after_used_bytes += other->_after_used_bytes;
624 _bytes_allocated_in_old_since_last_pause += other->_bytes_allocated_in_old_since_last_pause;
625 _failure_used_words += other->_failure_used_words;
626 _failure_waste_words += other->_failure_waste_words;
627 _regions_freed += other->_regions_freed;
628 }
629
630 void report(G1CollectedHeap* g1h, G1EvacInfo* evacuation_info) {
631 evacuation_info->set_regions_freed(_regions_freed);
632 evacuation_info->set_collection_set_used_before(_before_used_bytes + _after_used_bytes);
633 evacuation_info->increment_collection_set_used_after(_after_used_bytes);
634
635 g1h->decrement_summary_bytes(_before_used_bytes);
636 g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->add_failure_used_and_waste(_failure_used_words, _failure_waste_words);
637
638 G1Policy *policy = g1h->policy();
639 policy->old_gen_alloc_tracker()->add_allocated_non_humongous_bytes(_bytes_allocated_in_old_since_last_pause);
640
641 policy->cset_regions_freed();
642 }
643
644 void account_failed_region(G1HeapRegion* r) {
645 size_t used_words = r->live_bytes() / HeapWordSize;
646 _failure_used_words += used_words;
647 _failure_waste_words += G1HeapRegion::GrainWords - used_words;
648 _after_used_bytes += r->used();
649
650 // When moving a young gen region to old gen, we "allocate" that whole
651 // region there. This is in addition to any already evacuated objects.
652 // Notify the policy about that. Old gen regions do not cause an
653 // additional allocation: both the objects still in the region and the
654 // ones already moved are accounted for elsewhere.
655 if (r->is_young()) {
656 _bytes_allocated_in_old_since_last_pause += G1HeapRegion::GrainBytes;
657 }
658 }
659
660 void account_evacuated_region(G1HeapRegion* r) {
661 size_t used = r->used();
662 assert(used > 0, "region %u %s zero used", r->hrm_index(), r->get_short_type_str());
663 _before_used_bytes += used;
664 _regions_freed += 1;
665 }
666 };
667
668 // Closure applied to all regions in the collection set.
669 class FreeCSetClosure : public G1HeapRegionClosure {
670 // Helper to send JFR events for regions.
671 class JFREventForRegion {
672 EventGCPhaseParallel _event;
673
674 public:
675 JFREventForRegion(G1HeapRegion* region, uint worker_id) : _event() {
676 _event.set_gcId(GCId::current());
677 _event.set_gcWorkerId(worker_id);
678 if (region->is_young()) {
679 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::YoungFreeCSet));
680 } else {
681 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::NonYoungFreeCSet));
682 }
683 }
684
685 ~JFREventForRegion() {
686 _event.commit();
687 }
688 };
689
690 // Helper to do timing for region work.
691 class TimerForRegion {
692 Tickspan& _time;
693 Ticks _start_time;
694 public:
695 TimerForRegion(Tickspan& time) : _time(time), _start_time(Ticks::now()) { }
696 ~TimerForRegion() {
697 _time += Ticks::now() - _start_time;
698 }
699 };
700
701 // FreeCSetClosure members
702 G1CollectedHeap* _g1h;
703 const size_t* _surviving_young_words;
704 uint _worker_id;
705 Tickspan _young_time;
706 Tickspan _non_young_time;
707 FreeCSetStats* _stats;
708 G1EvacFailureRegions* _evac_failure_regions;
709 uint _num_retained_regions;
710
711 void assert_tracks_surviving_words(G1HeapRegion* r) {
712 assert(r->young_index_in_cset() != 0 &&
713 (uint)r->young_index_in_cset() <= _g1h->collection_set()->num_young_regions(),
714 "Young index %u is wrong for region %u of type %s with %u young regions",
715 r->young_index_in_cset(), r->hrm_index(), r->get_type_str(), _g1h->collection_set()->num_young_regions());
716 }
717
718 void handle_evacuated_region(G1HeapRegion* r) {
719 assert(!r->is_empty(), "Region %u is an empty region in the collection set.", r->hrm_index());
720 stats()->account_evacuated_region(r);
721
722 G1HeapRegionPrinter::evac_reclaim(r);
723 // Free the region and its remembered set.
724 _g1h->free_region(r, nullptr);
725 }
726
727 void handle_failed_region(G1HeapRegion* r) {
728 // Do some allocation statistics accounting. Regions that failed evacuation
729 // are always made old, so there is no need to update anything in the young
730 // gen statistics, but we need to update old gen statistics.
731 stats()->account_failed_region(r);
732
733 G1GCPhaseTimes* p = _g1h->phase_times();
734 assert(r->in_collection_set(), "Failed evacuation of region %u not in collection set", r->hrm_index());
735
736 p->record_or_add_thread_work_item(G1GCPhaseTimes::RestoreEvacuationFailedRegions,
737 _worker_id,
738 1,
739 G1GCPhaseTimes::RestoreEvacFailureRegionsEvacFailedNum);
740
741 bool retain_region = _g1h->policy()->should_retain_evac_failed_region(r);
742 // Update the region state due to the failed evacuation.
743 r->handle_evacuation_failure(retain_region);
744 assert(r->is_old(), "must already be relabelled as old");
745
746 if (retain_region) {
747 _g1h->retain_region(r);
748 _num_retained_regions++;
749 }
750 assert(retain_region == r->rem_set()->is_tracked(), "When retaining a region, remembered set should be kept.");
751
752 // Add region to old set, need to hold lock.
753 MutexLocker x(G1OldSets_lock, Mutex::_no_safepoint_check_flag);
754 _g1h->old_set_add(r);
755 }
756
757 Tickspan& timer_for_region(G1HeapRegion* r) {
758 return r->is_young() ? _young_time : _non_young_time;
759 }
760
761 FreeCSetStats* stats() {
762 return _stats;
763 }
764
765 public:
766 FreeCSetClosure(const size_t* surviving_young_words,
767 uint worker_id,
768 FreeCSetStats* stats,
769 G1EvacFailureRegions* evac_failure_regions) :
770 G1HeapRegionClosure(),
771 _g1h(G1CollectedHeap::heap()),
772 _surviving_young_words(surviving_young_words),
773 _worker_id(worker_id),
774 _young_time(),
775 _non_young_time(),
776 _stats(stats),
777 _evac_failure_regions(evac_failure_regions),
778 _num_retained_regions(0) { }
779
780 virtual bool do_heap_region(G1HeapRegion* r) {
781 assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
782 JFREventForRegion event(r, _worker_id);
783 TimerForRegion timer(timer_for_region(r));
784
785 if (r->is_young()) {
786 assert_tracks_surviving_words(r);
787 r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
788 }
789
790 if (_evac_failure_regions->contains(r->hrm_index())) {
791 handle_failed_region(r);
792 } else {
793 handle_evacuated_region(r);
794 }
795 assert(!_g1h->is_on_master_free_list(r), "sanity");
796
797 return false;
798 }
799
800 void report_timing() {
801 G1GCPhaseTimes* pt = _g1h->phase_times();
802 if (_young_time.value() > 0) {
803 pt->record_time_secs(G1GCPhaseTimes::YoungFreeCSet, _worker_id, _young_time.seconds());
804 }
805 if (_non_young_time.value() > 0) {
806 pt->record_time_secs(G1GCPhaseTimes::NonYoungFreeCSet, _worker_id, _non_young_time.seconds());
807 }
808 }
809
810 bool num_retained_regions() const { return _num_retained_regions; }
811 };
812
813 class G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask : public G1AbstractSubTask {
814 G1CollectedHeap* _g1h;
815 G1EvacInfo* _evacuation_info;
816 FreeCSetStats* _worker_stats;
817 G1HeapRegionClaimer _claimer;
818 const size_t* _surviving_young_words;
819 uint _active_workers;
820 G1EvacFailureRegions* _evac_failure_regions;
821 Atomic<uint> _num_retained_regions;
822
823 FreeCSetStats* worker_stats(uint worker) {
824 return &_worker_stats[worker];
825 }
826
827 void report_statistics() {
828 // Merge the accounting
829 FreeCSetStats total_stats;
830 for (uint worker = 0; worker < _active_workers; worker++) {
831 total_stats.merge_stats(worker_stats(worker));
832 }
833 total_stats.report(_g1h, _evacuation_info);
834 }
835
836 public:
837 FreeCollectionSetTask(G1EvacInfo* evacuation_info,
838 const size_t* surviving_young_words,
839 G1EvacFailureRegions* evac_failure_regions) :
840 G1AbstractSubTask(G1GCPhaseTimes::FreeCollectionSet),
841 _g1h(G1CollectedHeap::heap()),
842 _evacuation_info(evacuation_info),
843 _worker_stats(nullptr),
844 _claimer(0),
845 _surviving_young_words(surviving_young_words),
846 _active_workers(0),
847 _evac_failure_regions(evac_failure_regions),
848 _num_retained_regions(0) {
849
850 _g1h->clear_eden();
851 }
852
853 virtual ~FreeCollectionSetTask() {
854 Ticks serial_time = Ticks::now();
855
856 bool has_new_retained_regions = _num_retained_regions.load_relaxed() != 0;
857 if (has_new_retained_regions) {
858 G1CollectionSetCandidates* candidates = _g1h->collection_set()->candidates();
859 candidates->sort_by_efficiency();
860 }
861
862 report_statistics();
863 for (uint worker = 0; worker < _active_workers; worker++) {
864 _worker_stats[worker].~FreeCSetStats();
865 }
866 FREE_C_HEAP_ARRAY(_worker_stats);
867
868 _g1h->clear_collection_set();
869
870 G1GCPhaseTimes* p = _g1h->phase_times();
871 p->record_serial_free_cset_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
872 }
873
874 double worker_cost() const override { return G1CollectedHeap::heap()->collection_set()->num_initial_regions(); }
875
876 void set_max_workers(uint max_workers) override {
877 _active_workers = max_workers;
878 _worker_stats = NEW_C_HEAP_ARRAY(FreeCSetStats, max_workers, mtGC);
879 ::new (_worker_stats) FreeCSetStats[_active_workers]{};
880 _claimer.set_n_workers(_active_workers);
881 }
882
883 void do_work(uint worker_id) override {
884 FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id), _evac_failure_regions);
885 _g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
886 // Report per-region type timings.
887 cl.report_timing();
888
889 _num_retained_regions.add_then_fetch(cl.num_retained_regions(), memory_order_relaxed);
890 }
891 };
892
893 class G1PostEvacuateCollectionSetCleanupTask2::ResizeTLABsAndSwapCardTableTask : public G1AbstractSubTask {
894 G1JavaThreadsListClaimer _claimer;
895
896 // There is not much work per thread so the number of threads per worker is high.
897 static const uint ThreadsPerWorker = 250;
898
899 public:
900 ResizeTLABsAndSwapCardTableTask()
901 : G1AbstractSubTask(G1GCPhaseTimes::ResizeThreadLABs), _claimer(ThreadsPerWorker)
902 {
903 G1BarrierSet::g1_barrier_set()->swap_global_card_table();
904 }
905
906 void do_work(uint worker_id) override {
907
908 class ResizeAndSwapCardTableClosure : public ThreadClosure {
909 public:
910
911 void do_thread(Thread* thread) {
912 if (UseTLAB && ResizeTLAB) {
913 thread->tlab().resize();
914 }
915
916 G1BarrierSet::g1_barrier_set()->update_card_table_base(thread);
917 }
918 } resize_and_swap_cl;
919
920 _claimer.apply(&resize_and_swap_cl);
921 }
922
923 double worker_cost() const override {
924 return (double)_claimer.length() / ThreadsPerWorker;
925 }
926 };
927
928 class G1PostEvacuateCollectionSetCleanupTask2::DestroyPssTask : public G1AbstractSubTask {
929 G1ParScanThreadStateSet* _per_thread_states;
930
931 public:
932 DestroyPssTask(G1ParScanThreadStateSet* per_thread_states) :
933 G1AbstractSubTask(G1GCPhaseTimes::DestroyPSS),
934 _per_thread_states(per_thread_states) { }
935
936 double worker_cost() const override { return 1.0; }
937
938 void do_work(uint worker_id) override {
939 _per_thread_states->destroy_worker_states();
940 // This must be here after above destroyed the per-thread allocators.
941 G1CollectedHeap::heap()->partial_array_state_manager()->reset();
942 }
943 };
944
945 G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
946 G1EvacInfo* evacuation_info,
947 G1EvacFailureRegions* evac_failure_regions) :
948 G1BatchedTask("Post Evacuate Cleanup 2", G1CollectedHeap::heap()->phase_times())
949 {
950 #ifdef COMPILER2
951 add_serial_task(new UpdateDerivedPointersTask());
952 #endif // COMPILER2
953 if (G1CollectedHeap::heap()->has_humongous_reclaim_candidates()) {
954 add_serial_task(new EagerlyReclaimHumongousObjectsTask());
955 }
956 add_serial_task(new DestroyPssTask(per_thread_states));
957
958 if (evac_failure_regions->has_regions_evac_failed()) {
959 add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
960 }
961
962 add_parallel_task(new ResizeTLABsAndSwapCardTableTask());
963 add_parallel_task(new FreeCollectionSetTask(evacuation_info,
964 per_thread_states->surviving_young_words(),
965 evac_failure_regions));
966 }