1 /*
2 * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "compiler/oopMap.hpp"
28 #include "gc/g1/g1CardSetMemory.hpp"
29 #include "gc/g1/g1CardTableEntryClosure.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectionSetCandidates.inline.hpp"
32 #include "gc/g1/g1CollectorState.hpp"
33 #include "gc/g1/g1ConcurrentMark.inline.hpp"
34 #include "gc/g1/g1EvacFailureRegions.inline.hpp"
35 #include "gc/g1/g1EvacInfo.hpp"
36 #include "gc/g1/g1EvacStats.inline.hpp"
37 #include "gc/g1/g1HeapRegion.inline.hpp"
38 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
39 #include "gc/g1/g1OopClosures.inline.hpp"
40 #include "gc/g1/g1ParScanThreadState.hpp"
41 #include "gc/g1/g1RemSet.hpp"
42 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
43 #include "gc/shared/bufferNode.hpp"
44 #include "jfr/jfrEvents.hpp"
45 #include "oops/access.inline.hpp"
46 #include "oops/compressedOops.inline.hpp"
47 #include "oops/oop.inline.hpp"
48 #include "runtime/prefetch.hpp"
49 #include "runtime/threads.hpp"
50 #include "runtime/threadSMR.hpp"
51 #include "utilities/bitMap.inline.hpp"
52 #include "utilities/ticks.hpp"
53
54 class G1PostEvacuateCollectionSetCleanupTask1::MergePssTask : public G1AbstractSubTask {
55 G1ParScanThreadStateSet* _per_thread_states;
56
57 public:
58 MergePssTask(G1ParScanThreadStateSet* per_thread_states) :
59 G1AbstractSubTask(G1GCPhaseTimes::MergePSS),
60 _per_thread_states(per_thread_states) { }
61
62 double worker_cost() const override { return 1.0; }
63
64 void do_work(uint worker_id) override { _per_thread_states->flush_stats(); }
65 };
66
67 class G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask : public G1AbstractSubTask {
68 bool _evacuation_failed;
69 bool _allocation_failed;
70
71 public:
72 RecalculateUsedTask(bool evacuation_failed, bool allocation_failed) :
73 G1AbstractSubTask(G1GCPhaseTimes::RecalculateUsed),
74 _evacuation_failed(evacuation_failed),
75 _allocation_failed(allocation_failed) { }
76
77 double worker_cost() const override {
78 // If there is no evacuation failure, the work to perform is minimal.
79 return _evacuation_failed ? 1.0 : AlmostNoWork;
80 }
81
82 void do_work(uint worker_id) override {
83 G1CollectedHeap::heap()->update_used_after_gc(_evacuation_failed);
84 if (_allocation_failed) {
85 // Reset the G1GCAllocationFailureALot counters and flags
86 G1CollectedHeap::heap()->allocation_failure_injector()->reset();
87 }
88 }
89 };
90
91 class G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask : public G1AbstractSubTask {
92 public:
93 SampleCollectionSetCandidatesTask() : G1AbstractSubTask(G1GCPhaseTimes::SampleCollectionSetCandidates) { }
94
95 static bool should_execute() {
96 return G1CollectedHeap::heap()->should_sample_collection_set_candidates();
97 }
98
99 double worker_cost() const override {
100 return should_execute() ? 1.0 : AlmostNoWork;
101 }
102
103 void do_work(uint worker_id) override {
104 G1CollectedHeap* g1h = G1CollectedHeap::heap();
105
106 G1MonotonicArenaMemoryStats _total;
107 G1CollectionSetCandidates* candidates = g1h->collection_set()->candidates();
108 for (HeapRegion* r : *candidates) {
109 _total.add(r->rem_set()->card_set_memory_stats());
110 }
111 g1h->set_collection_set_candidates_stats(_total);
112 }
113 };
114
115 class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : public G1AbstractSubTask {
116 G1CollectedHeap* _g1h;
117 G1ConcurrentMark* _cm;
118
119 G1EvacFailureRegions* _evac_failure_regions;
120 CHeapBitMap _chunk_bitmap;
121
122 uint _num_chunks_per_region;
123 uint _num_evac_fail_regions;
124 size_t _chunk_size;
125
126 class PhaseTimesStat {
127 static constexpr G1GCPhaseTimes::GCParPhases phase_name =
128 G1GCPhaseTimes::RemoveSelfForwards;
129
130 G1GCPhaseTimes* _phase_times;
131 uint _worker_id;
132 Ticks _start;
133
134 public:
135 PhaseTimesStat(G1GCPhaseTimes* phase_times, uint worker_id) :
136 _phase_times(phase_times),
137 _worker_id(worker_id),
138 _start(Ticks::now()) { }
139
140 ~PhaseTimesStat() {
141 _phase_times->record_or_add_time_secs(phase_name,
142 _worker_id,
143 (Ticks::now() - _start).seconds());
144 }
145
146 void register_empty_chunk() {
147 _phase_times->record_or_add_thread_work_item(phase_name,
148 _worker_id,
149 1,
150 G1GCPhaseTimes::RemoveSelfForwardEmptyChunksNum);
151 }
152
153 void register_nonempty_chunk() {
154 _phase_times->record_or_add_thread_work_item(phase_name,
155 _worker_id,
156 1,
157 G1GCPhaseTimes::RemoveSelfForwardChunksNum);
158 }
159
160 void register_objects_count_and_size(size_t num_marked_obj, size_t marked_words) {
161 _phase_times->record_or_add_thread_work_item(phase_name,
162 _worker_id,
163 num_marked_obj,
164 G1GCPhaseTimes::RemoveSelfForwardObjectsNum);
165
166 size_t marked_bytes = marked_words * HeapWordSize;
167 _phase_times->record_or_add_thread_work_item(phase_name,
168 _worker_id,
169 marked_bytes,
170 G1GCPhaseTimes::RemoveSelfForwardObjectsBytes);
171 }
172 };
173
174 // Fill the memory area from start to end with filler objects, and update the BOT
175 // accordingly. Since we clear and use the bitmap for marking objects that failed
176 // evacuation, there is no other work to be done there.
177 static size_t zap_dead_objects(HeapRegion* hr, HeapWord* start, HeapWord* end) {
178 assert(start <= end, "precondition");
179 if (start == end) {
180 return 0;
181 }
182
183 hr->fill_range_with_dead_objects(start, end);
184 return pointer_delta(end, start);
185 }
186
187 static void update_garbage_words_in_hr(HeapRegion* hr, size_t garbage_words) {
188 if (garbage_words != 0) {
189 hr->note_self_forward_chunk_done(garbage_words * HeapWordSize);
190 }
191 }
192
193 static void prefetch_obj(HeapWord* obj_addr) {
194 Prefetch::write(obj_addr, PrefetchScanIntervalInBytes);
195 }
196
197 bool claim_chunk(uint chunk_idx) {
198 return _chunk_bitmap.par_set_bit(chunk_idx);
199 }
200
201 void process_chunk(uint worker_id, uint chunk_idx) {
202 PhaseTimesStat stat(_g1h->phase_times(), worker_id);
203
204 G1CMBitMap* bitmap = _cm->mark_bitmap();
205 const uint region_idx = _evac_failure_regions->get_region_idx(chunk_idx / _num_chunks_per_region);
206 HeapRegion* hr = _g1h->region_at(region_idx);
207
208 HeapWord* hr_bottom = hr->bottom();
209 HeapWord* hr_top = hr->top();
210 HeapWord* chunk_start = hr_bottom + (chunk_idx % _num_chunks_per_region) * _chunk_size;
211
212 assert(chunk_start < hr->end(), "inv");
213 if (chunk_start >= hr_top) {
214 return;
215 }
216
217 HeapWord* chunk_end = MIN2(chunk_start + _chunk_size, hr_top);
218 HeapWord* first_marked_addr = bitmap->get_next_marked_addr(chunk_start, hr_top);
219
220 size_t garbage_words = 0;
221
222 if (chunk_start == hr_bottom) {
223 // This is the bottom-most chunk in this region; zap [bottom, first_marked_addr).
224 garbage_words += zap_dead_objects(hr, hr_bottom, first_marked_addr);
225 }
226
227 if (first_marked_addr >= chunk_end) {
228 stat.register_empty_chunk();
229 update_garbage_words_in_hr(hr, garbage_words);
230 return;
231 }
232
233 stat.register_nonempty_chunk();
234
235 size_t num_marked_objs = 0;
236 size_t marked_words = 0;
237
238 HeapWord* obj_addr = first_marked_addr;
239 assert(chunk_start <= obj_addr && obj_addr < chunk_end,
240 "object " PTR_FORMAT " must be within chunk [" PTR_FORMAT ", " PTR_FORMAT "[",
241 p2i(obj_addr), p2i(chunk_start), p2i(chunk_end));
242 do {
243 assert(bitmap->is_marked(obj_addr), "inv");
244 prefetch_obj(obj_addr);
245
246 oop obj = cast_to_oop(obj_addr);
247 const size_t obj_size = obj->size();
248 HeapWord* const obj_end_addr = obj_addr + obj_size;
249
250 {
251 // Process marked object.
252 assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded");
253 obj->unset_self_forwarded();
254 hr->update_bot_for_block(obj_addr, obj_end_addr);
255
256 // Statistics
257 num_marked_objs++;
258 marked_words += obj_size;
259 }
260
261 assert(obj_end_addr <= hr_top, "inv");
262 // Use hr_top as the limit so that we zap dead ranges up to the next
263 // marked obj or hr_top.
264 HeapWord* next_marked_obj_addr = bitmap->get_next_marked_addr(obj_end_addr, hr_top);
265 garbage_words += zap_dead_objects(hr, obj_end_addr, next_marked_obj_addr);
266 obj_addr = next_marked_obj_addr;
267 } while (obj_addr < chunk_end);
268
269 assert(marked_words > 0 && num_marked_objs > 0, "inv");
270
271 stat.register_objects_count_and_size(num_marked_objs, marked_words);
272
273 update_garbage_words_in_hr(hr, garbage_words);
274 }
275
276 public:
277 RestoreEvacFailureRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
278 G1AbstractSubTask(G1GCPhaseTimes::RestoreEvacuationFailedRegions),
279 _g1h(G1CollectedHeap::heap()),
280 _cm(_g1h->concurrent_mark()),
281 _evac_failure_regions(evac_failure_regions),
282 _chunk_bitmap(mtGC) {
283
284 _num_evac_fail_regions = _evac_failure_regions->num_regions_evac_failed();
285 _num_chunks_per_region = G1CollectedHeap::get_chunks_per_region();
286
287 _chunk_size = static_cast<uint>(HeapRegion::GrainWords / _num_chunks_per_region);
288
289 log_debug(gc, ergo)("Initializing removing self forwards with %u chunks per region",
290 _num_chunks_per_region);
291
292 _chunk_bitmap.resize(_num_chunks_per_region * _num_evac_fail_regions);
293 }
294
295 double worker_cost() const override {
296 assert(_evac_failure_regions->has_regions_evac_failed(), "Should not call this if there were no evacuation failures");
297
298 double workers_per_region = (double)G1CollectedHeap::get_chunks_per_region() / G1RestoreRetainedRegionChunksPerWorker;
299 return workers_per_region * _evac_failure_regions->num_regions_evac_failed();
300 }
301
302 void do_work(uint worker_id) override {
303 const uint total_workers = G1CollectedHeap::heap()->workers()->active_workers();
304 const uint total_chunks = _num_chunks_per_region * _num_evac_fail_regions;
305 const uint start_chunk_idx = worker_id * total_chunks / total_workers;
306
307 for (uint i = 0; i < total_chunks; i++) {
308 const uint chunk_idx = (start_chunk_idx + i) % total_chunks;
309 if (claim_chunk(chunk_idx)) {
310 process_chunk(worker_id, chunk_idx);
311 }
312 }
313 }
314 };
315
316 G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
317 G1EvacFailureRegions* evac_failure_regions) :
318 G1BatchedTask("Post Evacuate Cleanup 1", G1CollectedHeap::heap()->phase_times())
319 {
320 bool evac_failed = evac_failure_regions->has_regions_evac_failed();
321 bool alloc_failed = evac_failure_regions->has_regions_alloc_failed();
322
323 add_serial_task(new MergePssTask(per_thread_states));
324 add_serial_task(new RecalculateUsedTask(evac_failed, alloc_failed));
325 if (SampleCollectionSetCandidatesTask::should_execute()) {
326 add_serial_task(new SampleCollectionSetCandidatesTask());
327 }
328 add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
329 if (evac_failed) {
330 add_parallel_task(new RestoreEvacFailureRegionsTask(evac_failure_regions));
331 }
332 }
333
334 class G1FreeHumongousRegionClosure : public HeapRegionIndexClosure {
335 uint _humongous_objects_reclaimed;
336 uint _humongous_regions_reclaimed;
337 size_t _freed_bytes;
338 G1CollectedHeap* _g1h;
339
340 // Returns whether the given humongous object defined by the start region index
341 // is reclaimable.
342 //
343 // At this point in the garbage collection, checking whether the humongous object
344 // is still a candidate is sufficient because:
345 //
346 // - if it has not been a candidate at the start of collection, it will never
347 // changed to be a candidate during the gc (and live).
348 // - any found outstanding (i.e. in the DCQ, or in its remembered set)
349 // references will set the candidate state to false.
350 // - there can be no references from within humongous starts regions referencing
351 // the object because we never allocate other objects into them.
352 // (I.e. there can be no intra-region references)
353 //
354 // It is not required to check whether the object has been found dead by marking
355 // or not, in fact it would prevent reclamation within a concurrent cycle, as
356 // all objects allocated during that time are considered live.
357 // SATB marking is even more conservative than the remembered set.
358 // So if at this point in the collection we did not find a reference during gc
359 // (or it had enough references to not be a candidate, having many remembered
360 // set entries), nobody has a reference to it.
361 // At the start of collection we flush all refinement logs, and remembered sets
362 // are completely up-to-date wrt to references to the humongous object.
363 //
364 // So there is no need to re-check remembered set size of the humongous region.
365 //
366 // Other implementation considerations:
367 // - never consider object arrays at this time because they would pose
368 // considerable effort for cleaning up the remembered sets. This is
369 // required because stale remembered sets might reference locations that
370 // are currently allocated into.
371 bool is_reclaimable(uint region_idx) const {
372 return G1CollectedHeap::heap()->is_humongous_reclaim_candidate(region_idx);
373 }
374
375 public:
376 G1FreeHumongousRegionClosure() :
377 _humongous_objects_reclaimed(0),
378 _humongous_regions_reclaimed(0),
379 _freed_bytes(0),
380 _g1h(G1CollectedHeap::heap())
381 {}
382
383 bool do_heap_region_index(uint region_index) override {
384 if (!is_reclaimable(region_index)) {
385 return false;
386 }
387
388 HeapRegion* r = _g1h->region_at(region_index);
389
390 oop obj = cast_to_oop(r->bottom());
391 guarantee(obj->is_typeArray(),
392 "Only eagerly reclaiming type arrays is supported, but the object "
393 PTR_FORMAT " is not.", p2i(r->bottom()));
394
395 log_debug(gc, humongous)("Reclaimed humongous region %u (object size " SIZE_FORMAT " @ " PTR_FORMAT ")",
396 region_index,
397 obj->size() * HeapWordSize,
398 p2i(r->bottom())
399 );
400
401 G1ConcurrentMark* const cm = _g1h->concurrent_mark();
402 cm->humongous_object_eagerly_reclaimed(r);
403 assert(!cm->is_marked_in_bitmap(obj),
404 "Eagerly reclaimed humongous region %u should not be marked at all but is in bitmap %s",
405 region_index,
406 BOOL_TO_STR(cm->is_marked_in_bitmap(obj)));
407 _humongous_objects_reclaimed++;
408
409 auto free_humongous_region = [&] (HeapRegion* r) {
410 _freed_bytes += r->used();
411 r->set_containing_set(nullptr);
412 _humongous_regions_reclaimed++;
413 _g1h->free_humongous_region(r, nullptr);
414 _g1h->hr_printer()->cleanup(r);
415 };
416
417 _g1h->humongous_obj_regions_iterate(r, free_humongous_region);
418
419 return false;
420 }
421
422 uint humongous_objects_reclaimed() {
423 return _humongous_objects_reclaimed;
424 }
425
426 uint humongous_regions_reclaimed() {
427 return _humongous_regions_reclaimed;
428 }
429
430 size_t bytes_freed() const {
431 return _freed_bytes;
432 }
433 };
434
435 #if COMPILER2_OR_JVMCI
436 class G1PostEvacuateCollectionSetCleanupTask2::UpdateDerivedPointersTask : public G1AbstractSubTask {
437 public:
438 UpdateDerivedPointersTask() : G1AbstractSubTask(G1GCPhaseTimes::UpdateDerivedPointers) { }
439
440 double worker_cost() const override { return 1.0; }
441 void do_work(uint worker_id) override { DerivedPointerTable::update_pointers(); }
442 };
443 #endif
444
445 class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask : public G1AbstractSubTask {
446 uint _humongous_regions_reclaimed;
447 size_t _bytes_freed;
448
449 public:
450 EagerlyReclaimHumongousObjectsTask() :
451 G1AbstractSubTask(G1GCPhaseTimes::EagerlyReclaimHumongousObjects),
452 _humongous_regions_reclaimed(0),
453 _bytes_freed(0) { }
454
455 virtual ~EagerlyReclaimHumongousObjectsTask() {
456 G1CollectedHeap* g1h = G1CollectedHeap::heap();
457
458 g1h->remove_from_old_gen_sets(0, _humongous_regions_reclaimed);
459 g1h->decrement_summary_bytes(_bytes_freed);
460 }
461
462 double worker_cost() const override { return 1.0; }
463 void do_work(uint worker_id) override {
464 G1CollectedHeap* g1h = G1CollectedHeap::heap();
465
466 G1FreeHumongousRegionClosure cl;
467 g1h->heap_region_iterate(&cl);
468
469 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumTotal, g1h->num_humongous_objects());
470 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumCandidates, g1h->num_humongous_reclaim_candidates());
471 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumReclaimed, cl.humongous_objects_reclaimed());
472
473 _humongous_regions_reclaimed = cl.humongous_regions_reclaimed();
474 _bytes_freed = cl.bytes_freed();
475 }
476 };
477
478 class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
479 size_t _num_dirtied;
480 G1CollectedHeap* _g1h;
481 G1CardTable* _g1_ct;
482 G1EvacFailureRegions* _evac_failure_regions;
483
484 HeapRegion* region_for_card(CardValue* card_ptr) const {
485 return _g1h->heap_region_containing(_g1_ct->addr_for(card_ptr));
486 }
487
488 bool will_become_free(HeapRegion* hr) const {
489 // A region will be freed by during the FreeCollectionSet phase if the region is in the
490 // collection set and has not had an evacuation failure.
491 return _g1h->is_in_cset(hr) && !_evac_failure_regions->contains(hr->hrm_index());
492 }
493
494 public:
495 RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h, G1EvacFailureRegions* evac_failure_regions) :
496 G1CardTableEntryClosure(),
497 _num_dirtied(0),
498 _g1h(g1h),
499 _g1_ct(g1h->card_table()),
500 _evac_failure_regions(evac_failure_regions) { }
501
502 void do_card_ptr(CardValue* card_ptr, uint worker_id) {
503 HeapRegion* hr = region_for_card(card_ptr);
504
505 // Should only dirty cards in regions that won't be freed.
506 if (!will_become_free(hr)) {
507 *card_ptr = G1CardTable::dirty_card_val();
508 _num_dirtied++;
509 }
510 }
511
512 size_t num_dirtied() const { return _num_dirtied; }
513 };
514
515 class G1PostEvacuateCollectionSetCleanupTask2::ProcessEvacuationFailedRegionsTask : public G1AbstractSubTask {
516 G1EvacFailureRegions* _evac_failure_regions;
517 HeapRegionClaimer _claimer;
518
519 class ProcessEvacuationFailedRegionsClosure : public HeapRegionClosure {
520 public:
521
522 bool do_heap_region(HeapRegion* r) override {
523 G1CollectedHeap* g1h = G1CollectedHeap::heap();
524 G1ConcurrentMark* cm = g1h->concurrent_mark();
525
526 HeapWord* top_at_mark_start = cm->top_at_mark_start(r);
527 assert(top_at_mark_start == r->bottom(), "TAMS must not have been set for region %u", r->hrm_index());
528 assert(cm->live_bytes(r->hrm_index()) == 0, "Marking live bytes must not be set for region %u", r->hrm_index());
529
530 // Concurrent mark does not mark through regions that we retain (they are root
531 // regions wrt to marking), so we must clear their mark data (tams, bitmap, ...)
532 // set eagerly or during evacuation failure.
533 bool clear_mark_data = !g1h->collector_state()->in_concurrent_start_gc() ||
534 g1h->policy()->should_retain_evac_failed_region(r);
535
536 if (clear_mark_data) {
537 g1h->clear_bitmap_for_region(r);
538 } else {
539 // This evacuation failed region is going to be marked through. Update mark data.
540 cm->update_top_at_mark_start(r);
541 cm->set_live_bytes(r->hrm_index(), r->live_bytes());
542 assert(cm->mark_bitmap()->get_next_marked_addr(r->bottom(), cm->top_at_mark_start(r)) != cm->top_at_mark_start(r),
543 "Marks must be on bitmap for region %u", r->hrm_index());
544 }
545 return false;
546 }
547 };
548
549 public:
550 ProcessEvacuationFailedRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
551 G1AbstractSubTask(G1GCPhaseTimes::ProcessEvacuationFailedRegions),
552 _evac_failure_regions(evac_failure_regions),
553 _claimer(0) {
554 }
555
556 void set_max_workers(uint max_workers) override {
557 _claimer.set_n_workers(max_workers);
558 }
559
560 double worker_cost() const override {
561 return _evac_failure_regions->num_regions_evac_failed();
562 }
563
564 void do_work(uint worker_id) override {
565 ProcessEvacuationFailedRegionsClosure cl;
566 _evac_failure_regions->par_iterate(&cl, &_claimer, worker_id);
567 }
568 };
569
570 class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G1AbstractSubTask {
571 BufferNodeList* _rdc_buffers;
572 uint _num_buffer_lists;
573 G1EvacFailureRegions* _evac_failure_regions;
574
575 public:
576 RedirtyLoggedCardsTask(G1EvacFailureRegions* evac_failure_regions, BufferNodeList* rdc_buffers, uint num_buffer_lists) :
577 G1AbstractSubTask(G1GCPhaseTimes::RedirtyCards),
578 _rdc_buffers(rdc_buffers),
579 _num_buffer_lists(num_buffer_lists),
580 _evac_failure_regions(evac_failure_regions) { }
581
582 double worker_cost() const override {
583 // Needs more investigation.
584 return G1CollectedHeap::heap()->workers()->active_workers();
585 }
586
587 void do_work(uint worker_id) override {
588 RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap(), _evac_failure_regions);
589
590 uint start = worker_id;
591 for (uint i = 0; i < _num_buffer_lists; i++) {
592 uint index = (start + i) % _num_buffer_lists;
593
594 BufferNode* next = Atomic::load(&_rdc_buffers[index]._head);
595 BufferNode* tail = Atomic::load(&_rdc_buffers[index]._tail);
596
597 while (next != nullptr) {
598 BufferNode* node = next;
599 next = Atomic::cmpxchg(&_rdc_buffers[index]._head, node, (node != tail ) ? node->next() : nullptr);
600 if (next == node) {
601 cl.apply_to_buffer(node, worker_id);
602 next = (node != tail ) ? node->next() : nullptr;
603 } else {
604 break; // If there is contention, move to the next BufferNodeList
605 }
606 }
607 }
608 record_work_item(worker_id, 0, cl.num_dirtied());
609 }
610 };
611
612 // Helper class to keep statistics for the collection set freeing
613 class FreeCSetStats {
614 size_t _before_used_bytes; // Usage in regions successfully evacuate
615 size_t _after_used_bytes; // Usage in regions failing evacuation
616 size_t _bytes_allocated_in_old_since_last_gc; // Size of young regions turned into old
617 size_t _failure_used_words; // Live size in failed regions
618 size_t _failure_waste_words; // Wasted size in failed regions
619 size_t _card_rs_length; // (Card Set) Remembered set size
620 uint _regions_freed; // Number of regions freed
621
622 public:
623 FreeCSetStats() :
624 _before_used_bytes(0),
625 _after_used_bytes(0),
626 _bytes_allocated_in_old_since_last_gc(0),
627 _failure_used_words(0),
628 _failure_waste_words(0),
629 _card_rs_length(0),
630 _regions_freed(0) { }
631
632 void merge_stats(FreeCSetStats* other) {
633 assert(other != nullptr, "invariant");
634 _before_used_bytes += other->_before_used_bytes;
635 _after_used_bytes += other->_after_used_bytes;
636 _bytes_allocated_in_old_since_last_gc += other->_bytes_allocated_in_old_since_last_gc;
637 _failure_used_words += other->_failure_used_words;
638 _failure_waste_words += other->_failure_waste_words;
639 _card_rs_length += other->_card_rs_length;
640 _regions_freed += other->_regions_freed;
641 }
642
643 void report(G1CollectedHeap* g1h, G1EvacInfo* evacuation_info) {
644 evacuation_info->set_regions_freed(_regions_freed);
645 evacuation_info->set_collection_set_used_before(_before_used_bytes + _after_used_bytes);
646 evacuation_info->increment_collection_set_used_after(_after_used_bytes);
647
648 g1h->decrement_summary_bytes(_before_used_bytes);
649 g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->add_failure_used_and_waste(_failure_used_words, _failure_waste_words);
650
651 G1Policy *policy = g1h->policy();
652 policy->old_gen_alloc_tracker()->add_allocated_bytes_since_last_gc(_bytes_allocated_in_old_since_last_gc);
653 policy->record_card_rs_length(_card_rs_length);
654 policy->cset_regions_freed();
655 }
656
657 void account_failed_region(HeapRegion* r) {
658 size_t used_words = r->live_bytes() / HeapWordSize;
659 _failure_used_words += used_words;
660 _failure_waste_words += HeapRegion::GrainWords - used_words;
661 _after_used_bytes += r->used();
662
663 // When moving a young gen region to old gen, we "allocate" that whole
664 // region there. This is in addition to any already evacuated objects.
665 // Notify the policy about that. Old gen regions do not cause an
666 // additional allocation: both the objects still in the region and the
667 // ones already moved are accounted for elsewhere.
668 if (r->is_young()) {
669 _bytes_allocated_in_old_since_last_gc += HeapRegion::GrainBytes;
670 }
671 }
672
673 void account_evacuated_region(HeapRegion* r) {
674 size_t used = r->used();
675 assert(used > 0, "region %u %s zero used", r->hrm_index(), r->get_short_type_str());
676 _before_used_bytes += used;
677 _regions_freed += 1;
678 }
679
680 void account_card_rs_length(HeapRegion* r) {
681 _card_rs_length += r->rem_set()->occupied();
682 }
683 };
684
685 // Closure applied to all regions in the collection set.
686 class FreeCSetClosure : public HeapRegionClosure {
687 // Helper to send JFR events for regions.
688 class JFREventForRegion {
689 EventGCPhaseParallel _event;
690
691 public:
692 JFREventForRegion(HeapRegion* region, uint worker_id) : _event() {
693 _event.set_gcId(GCId::current());
694 _event.set_gcWorkerId(worker_id);
695 if (region->is_young()) {
696 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::YoungFreeCSet));
697 } else {
698 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::NonYoungFreeCSet));
699 }
700 }
701
702 ~JFREventForRegion() {
703 _event.commit();
704 }
705 };
706
707 // Helper to do timing for region work.
708 class TimerForRegion {
709 Tickspan& _time;
710 Ticks _start_time;
711 public:
712 TimerForRegion(Tickspan& time) : _time(time), _start_time(Ticks::now()) { }
713 ~TimerForRegion() {
714 _time += Ticks::now() - _start_time;
715 }
716 };
717
718 // FreeCSetClosure members
719 G1CollectedHeap* _g1h;
720 const size_t* _surviving_young_words;
721 uint _worker_id;
722 Tickspan _young_time;
723 Tickspan _non_young_time;
724 FreeCSetStats* _stats;
725 G1EvacFailureRegions* _evac_failure_regions;
726 uint _num_retained_regions;
727
728 void assert_tracks_surviving_words(HeapRegion* r) {
729 assert(r->young_index_in_cset() != 0 &&
730 (uint)r->young_index_in_cset() <= _g1h->collection_set()->young_region_length(),
731 "Young index %u is wrong for region %u of type %s with %u young regions",
732 r->young_index_in_cset(), r->hrm_index(), r->get_type_str(), _g1h->collection_set()->young_region_length());
733 }
734
735 void handle_evacuated_region(HeapRegion* r) {
736 assert(!r->is_empty(), "Region %u is an empty region in the collection set.", r->hrm_index());
737 stats()->account_evacuated_region(r);
738
739 // Free the region and its remembered set.
740 _g1h->free_region(r, nullptr);
741 _g1h->hr_printer()->cleanup(r);
742 }
743
744 void handle_failed_region(HeapRegion* r) {
745 // Do some allocation statistics accounting. Regions that failed evacuation
746 // are always made old, so there is no need to update anything in the young
747 // gen statistics, but we need to update old gen statistics.
748 stats()->account_failed_region(r);
749
750 G1GCPhaseTimes* p = _g1h->phase_times();
751 assert(r->in_collection_set(), "Failed evacuation of region %u not in collection set", r->hrm_index());
752
753 p->record_or_add_thread_work_item(G1GCPhaseTimes::RestoreEvacuationFailedRegions,
754 _worker_id,
755 1,
756 G1GCPhaseTimes::RestoreEvacFailureRegionsEvacFailedNum);
757
758 bool retain_region = _g1h->policy()->should_retain_evac_failed_region(r);
759 // Update the region state due to the failed evacuation.
760 r->handle_evacuation_failure(retain_region);
761 assert(r->is_old(), "must already be relabelled as old");
762
763 if (retain_region) {
764 _g1h->retain_region(r);
765 _num_retained_regions++;
766 }
767 assert(retain_region == r->rem_set()->is_tracked(), "When retaining a region, remembered set should be kept.");
768
769 // Add region to old set, need to hold lock.
770 MutexLocker x(OldSets_lock, Mutex::_no_safepoint_check_flag);
771 _g1h->old_set_add(r);
772 }
773
774 Tickspan& timer_for_region(HeapRegion* r) {
775 return r->is_young() ? _young_time : _non_young_time;
776 }
777
778 FreeCSetStats* stats() {
779 return _stats;
780 }
781
782 public:
783 FreeCSetClosure(const size_t* surviving_young_words,
784 uint worker_id,
785 FreeCSetStats* stats,
786 G1EvacFailureRegions* evac_failure_regions) :
787 HeapRegionClosure(),
788 _g1h(G1CollectedHeap::heap()),
789 _surviving_young_words(surviving_young_words),
790 _worker_id(worker_id),
791 _young_time(),
792 _non_young_time(),
793 _stats(stats),
794 _evac_failure_regions(evac_failure_regions),
795 _num_retained_regions(0) { }
796
797 virtual bool do_heap_region(HeapRegion* r) {
798 assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
799 JFREventForRegion event(r, _worker_id);
800 TimerForRegion timer(timer_for_region(r));
801
802 stats()->account_card_rs_length(r);
803
804 if (r->is_young()) {
805 assert_tracks_surviving_words(r);
806 r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
807 }
808
809 if (_evac_failure_regions->contains(r->hrm_index())) {
810 handle_failed_region(r);
811 } else {
812 handle_evacuated_region(r);
813 }
814 assert(!_g1h->is_on_master_free_list(r), "sanity");
815
816 return false;
817 }
818
819 void report_timing() {
820 G1GCPhaseTimes* pt = _g1h->phase_times();
821 if (_young_time.value() > 0) {
822 pt->record_time_secs(G1GCPhaseTimes::YoungFreeCSet, _worker_id, _young_time.seconds());
823 }
824 if (_non_young_time.value() > 0) {
825 pt->record_time_secs(G1GCPhaseTimes::NonYoungFreeCSet, _worker_id, _non_young_time.seconds());
826 }
827 }
828
829 bool num_retained_regions() const { return _num_retained_regions; }
830 };
831
832 class G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask : public G1AbstractSubTask {
833 G1CollectedHeap* _g1h;
834 G1EvacInfo* _evacuation_info;
835 FreeCSetStats* _worker_stats;
836 HeapRegionClaimer _claimer;
837 const size_t* _surviving_young_words;
838 uint _active_workers;
839 G1EvacFailureRegions* _evac_failure_regions;
840 volatile uint _num_retained_regions;
841
842 FreeCSetStats* worker_stats(uint worker) {
843 return &_worker_stats[worker];
844 }
845
846 void report_statistics() {
847 // Merge the accounting
848 FreeCSetStats total_stats;
849 for (uint worker = 0; worker < _active_workers; worker++) {
850 total_stats.merge_stats(worker_stats(worker));
851 }
852 total_stats.report(_g1h, _evacuation_info);
853 }
854
855 public:
856 FreeCollectionSetTask(G1EvacInfo* evacuation_info,
857 const size_t* surviving_young_words,
858 G1EvacFailureRegions* evac_failure_regions) :
859 G1AbstractSubTask(G1GCPhaseTimes::FreeCollectionSet),
860 _g1h(G1CollectedHeap::heap()),
861 _evacuation_info(evacuation_info),
862 _worker_stats(nullptr),
863 _claimer(0),
864 _surviving_young_words(surviving_young_words),
865 _active_workers(0),
866 _evac_failure_regions(evac_failure_regions),
867 _num_retained_regions(0) {
868
869 _g1h->clear_eden();
870 }
871
872 virtual ~FreeCollectionSetTask() {
873 Ticks serial_time = Ticks::now();
874
875 bool has_new_retained_regions = Atomic::load(&_num_retained_regions) != 0;
876 if (has_new_retained_regions) {
877 G1CollectionSetCandidates* candidates = _g1h->collection_set()->candidates();
878 candidates->sort_by_efficiency();
879 }
880
881 report_statistics();
882 for (uint worker = 0; worker < _active_workers; worker++) {
883 _worker_stats[worker].~FreeCSetStats();
884 }
885 FREE_C_HEAP_ARRAY(FreeCSetStats, _worker_stats);
886
887 G1GCPhaseTimes* p = _g1h->phase_times();
888 p->record_serial_free_cset_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
889
890 _g1h->clear_collection_set();
891 }
892
893 double worker_cost() const override { return G1CollectedHeap::heap()->collection_set()->region_length(); }
894
895 void set_max_workers(uint max_workers) override {
896 _active_workers = max_workers;
897 _worker_stats = NEW_C_HEAP_ARRAY(FreeCSetStats, max_workers, mtGC);
898 for (uint worker = 0; worker < _active_workers; worker++) {
899 ::new (&_worker_stats[worker]) FreeCSetStats();
900 }
901 _claimer.set_n_workers(_active_workers);
902 }
903
904 void do_work(uint worker_id) override {
905 FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id), _evac_failure_regions);
906 _g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
907 // Report per-region type timings.
908 cl.report_timing();
909
910 Atomic::add(&_num_retained_regions, cl.num_retained_regions(), memory_order_relaxed);
911 }
912 };
913
914 class G1PostEvacuateCollectionSetCleanupTask2::ResizeTLABsTask : public G1AbstractSubTask {
915 G1JavaThreadsListClaimer _claimer;
916
917 // There is not much work per thread so the number of threads per worker is high.
918 static const uint ThreadsPerWorker = 250;
919
920 public:
921 ResizeTLABsTask() : G1AbstractSubTask(G1GCPhaseTimes::ResizeThreadLABs), _claimer(ThreadsPerWorker) { }
922
923 void do_work(uint worker_id) override {
924 class ResizeClosure : public ThreadClosure {
925 public:
926
927 void do_thread(Thread* thread) {
928 static_cast<JavaThread*>(thread)->tlab().resize();
929 }
930 } cl;
931 _claimer.apply(&cl);
932 }
933
934 double worker_cost() const override {
935 return (double)_claimer.length() / ThreadsPerWorker;
936 }
937 };
938
939 G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
940 G1EvacInfo* evacuation_info,
941 G1EvacFailureRegions* evac_failure_regions) :
942 G1BatchedTask("Post Evacuate Cleanup 2", G1CollectedHeap::heap()->phase_times())
943 {
944 #if COMPILER2_OR_JVMCI
945 add_serial_task(new UpdateDerivedPointersTask());
946 #endif
947 if (G1CollectedHeap::heap()->has_humongous_reclaim_candidates()) {
948 add_serial_task(new EagerlyReclaimHumongousObjectsTask());
949 }
950
951 if (evac_failure_regions->has_regions_evac_failed()) {
952 add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
953 }
954 add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,
955 per_thread_states->rdc_buffers(),
956 per_thread_states->num_workers()));
957
958 if (UseTLAB && ResizeTLAB) {
959 add_parallel_task(new ResizeTLABsTask());
960 }
961 add_parallel_task(new FreeCollectionSetTask(evacuation_info,
962 per_thread_states->surviving_young_words(),
963 evac_failure_regions));
964 }