1 /*
2 * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "compiler/oopMap.hpp"
28 #include "gc/g1/g1CardSetMemory.hpp"
29 #include "gc/g1/g1CardTableEntryClosure.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectionSetCandidates.inline.hpp"
32 #include "gc/g1/g1CollectorState.hpp"
33 #include "gc/g1/g1ConcurrentMark.inline.hpp"
34 #include "gc/g1/g1EvacFailureRegions.inline.hpp"
35 #include "gc/g1/g1EvacInfo.hpp"
36 #include "gc/g1/g1EvacStats.inline.hpp"
37 #include "gc/g1/g1HeapRegion.inline.hpp"
38 #include "gc/g1/g1HeapRegionPrinter.hpp"
39 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
40 #include "gc/g1/g1OopClosures.inline.hpp"
41 #include "gc/g1/g1ParScanThreadState.hpp"
42 #include "gc/g1/g1RemSet.hpp"
43 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
44 #include "gc/shared/bufferNode.hpp"
45 #include "gc/shared/preservedMarks.inline.hpp"
46 #include "jfr/jfrEvents.hpp"
47 #include "oops/access.inline.hpp"
48 #include "oops/compressedOops.inline.hpp"
49 #include "oops/oop.inline.hpp"
50 #include "runtime/prefetch.hpp"
51 #include "runtime/threads.hpp"
52 #include "runtime/threadSMR.hpp"
53 #include "utilities/bitMap.inline.hpp"
54 #include "utilities/ticks.hpp"
55
56 class G1PostEvacuateCollectionSetCleanupTask1::MergePssTask : public G1AbstractSubTask {
57 G1ParScanThreadStateSet* _per_thread_states;
58
59 public:
60 MergePssTask(G1ParScanThreadStateSet* per_thread_states) :
61 G1AbstractSubTask(G1GCPhaseTimes::MergePSS),
62 _per_thread_states(per_thread_states) { }
63
64 double worker_cost() const override { return 1.0; }
65
66 void do_work(uint worker_id) override { _per_thread_states->flush_stats(); }
67 };
68
69 class G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask : public G1AbstractSubTask {
70 bool _evacuation_failed;
71 bool _allocation_failed;
72
73 public:
74 RecalculateUsedTask(bool evacuation_failed, bool allocation_failed) :
75 G1AbstractSubTask(G1GCPhaseTimes::RecalculateUsed),
76 _evacuation_failed(evacuation_failed),
77 _allocation_failed(allocation_failed) { }
78
79 double worker_cost() const override {
80 // If there is no evacuation failure, the work to perform is minimal.
81 return _evacuation_failed ? 1.0 : AlmostNoWork;
82 }
83
84 void do_work(uint worker_id) override {
85 G1CollectedHeap::heap()->update_used_after_gc(_evacuation_failed);
86 if (_allocation_failed) {
87 // Reset the G1GCAllocationFailureALot counters and flags
88 G1CollectedHeap::heap()->allocation_failure_injector()->reset();
89 }
90 }
91 };
92
93 class G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask : public G1AbstractSubTask {
94 public:
95 SampleCollectionSetCandidatesTask() : G1AbstractSubTask(G1GCPhaseTimes::SampleCollectionSetCandidates) { }
96
97 static bool should_execute() {
98 return G1CollectedHeap::heap()->should_sample_collection_set_candidates();
99 }
100
101 double worker_cost() const override {
102 return should_execute() ? 1.0 : AlmostNoWork;
103 }
104
105 void do_work(uint worker_id) override {
106 G1CollectedHeap* g1h = G1CollectedHeap::heap();
107
108 G1MonotonicArenaMemoryStats _total;
109 G1CollectionSetCandidates* candidates = g1h->collection_set()->candidates();
110 for (G1HeapRegion* r : *candidates) {
111 _total.add(r->rem_set()->card_set_memory_stats());
112 }
113 g1h->set_collection_set_candidates_stats(_total);
114 }
115 };
116
117 class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : public G1AbstractSubTask {
118 G1CollectedHeap* _g1h;
119 G1ConcurrentMark* _cm;
120
121 G1EvacFailureRegions* _evac_failure_regions;
122 CHeapBitMap _chunk_bitmap;
123
124 uint _num_chunks_per_region;
125 uint _num_evac_fail_regions;
126 size_t _chunk_size;
127
128 class PhaseTimesStat {
129 static constexpr G1GCPhaseTimes::GCParPhases phase_name =
130 G1GCPhaseTimes::RemoveSelfForwards;
131
132 G1GCPhaseTimes* _phase_times;
133 uint _worker_id;
134 Ticks _start;
135
136 public:
137 PhaseTimesStat(G1GCPhaseTimes* phase_times, uint worker_id) :
138 _phase_times(phase_times),
139 _worker_id(worker_id),
140 _start(Ticks::now()) { }
141
142 ~PhaseTimesStat() {
143 _phase_times->record_or_add_time_secs(phase_name,
144 _worker_id,
145 (Ticks::now() - _start).seconds());
146 }
147
148 void register_empty_chunk() {
149 _phase_times->record_or_add_thread_work_item(phase_name,
150 _worker_id,
151 1,
152 G1GCPhaseTimes::RemoveSelfForwardEmptyChunksNum);
153 }
154
155 void register_nonempty_chunk() {
156 _phase_times->record_or_add_thread_work_item(phase_name,
157 _worker_id,
158 1,
159 G1GCPhaseTimes::RemoveSelfForwardChunksNum);
160 }
161
162 void register_objects_count_and_size(size_t num_marked_obj, size_t marked_words) {
163 _phase_times->record_or_add_thread_work_item(phase_name,
164 _worker_id,
165 num_marked_obj,
166 G1GCPhaseTimes::RemoveSelfForwardObjectsNum);
167
168 size_t marked_bytes = marked_words * HeapWordSize;
169 _phase_times->record_or_add_thread_work_item(phase_name,
170 _worker_id,
171 marked_bytes,
172 G1GCPhaseTimes::RemoveSelfForwardObjectsBytes);
173 }
174 };
175
176 // Fill the memory area from start to end with filler objects, and update the BOT
177 // accordingly. Since we clear and use the bitmap for marking objects that failed
178 // evacuation, there is no other work to be done there.
179 static size_t zap_dead_objects(G1HeapRegion* hr, HeapWord* start, HeapWord* end) {
180 assert(start <= end, "precondition");
181 if (start == end) {
182 return 0;
183 }
184
185 hr->fill_range_with_dead_objects(start, end);
186 return pointer_delta(end, start);
187 }
188
189 static void update_garbage_words_in_hr(G1HeapRegion* hr, size_t garbage_words) {
190 if (garbage_words != 0) {
191 hr->note_self_forward_chunk_done(garbage_words * HeapWordSize);
192 }
193 }
194
195 static void prefetch_obj(HeapWord* obj_addr) {
196 Prefetch::write(obj_addr, PrefetchScanIntervalInBytes);
197 }
198
199 bool claim_chunk(uint chunk_idx) {
200 return _chunk_bitmap.par_set_bit(chunk_idx);
201 }
202
203 void process_chunk(uint worker_id, uint chunk_idx) {
204 PhaseTimesStat stat(_g1h->phase_times(), worker_id);
205
206 G1CMBitMap* bitmap = _cm->mark_bitmap();
207 const uint region_idx = _evac_failure_regions->get_region_idx(chunk_idx / _num_chunks_per_region);
208 G1HeapRegion* hr = _g1h->region_at(region_idx);
209
210 HeapWord* hr_bottom = hr->bottom();
211 HeapWord* hr_top = hr->top();
212 HeapWord* chunk_start = hr_bottom + (chunk_idx % _num_chunks_per_region) * _chunk_size;
213
214 assert(chunk_start < hr->end(), "inv");
215 if (chunk_start >= hr_top) {
216 return;
217 }
218
219 HeapWord* chunk_end = MIN2(chunk_start + _chunk_size, hr_top);
220 HeapWord* first_marked_addr = bitmap->get_next_marked_addr(chunk_start, hr_top);
221
222 size_t garbage_words = 0;
223
224 if (chunk_start == hr_bottom) {
225 // This is the bottom-most chunk in this region; zap [bottom, first_marked_addr).
226 garbage_words += zap_dead_objects(hr, hr_bottom, first_marked_addr);
227 }
228
229 if (first_marked_addr >= chunk_end) {
230 stat.register_empty_chunk();
231 update_garbage_words_in_hr(hr, garbage_words);
232 return;
233 }
234
235 stat.register_nonempty_chunk();
236
237 size_t num_marked_objs = 0;
238 size_t marked_words = 0;
239
240 HeapWord* obj_addr = first_marked_addr;
241 assert(chunk_start <= obj_addr && obj_addr < chunk_end,
242 "object " PTR_FORMAT " must be within chunk [" PTR_FORMAT ", " PTR_FORMAT "[",
243 p2i(obj_addr), p2i(chunk_start), p2i(chunk_end));
244 do {
245 assert(bitmap->is_marked(obj_addr), "inv");
246 prefetch_obj(obj_addr);
247
248 oop obj = cast_to_oop(obj_addr);
249 const size_t obj_size = obj->size();
250 HeapWord* const obj_end_addr = obj_addr + obj_size;
251
252 {
253 // Process marked object.
254 assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded");
255 obj->init_mark();
256 hr->update_bot_for_block(obj_addr, obj_end_addr);
257
258 // Statistics
259 num_marked_objs++;
260 marked_words += obj_size;
261 }
262
263 assert(obj_end_addr <= hr_top, "inv");
264 // Use hr_top as the limit so that we zap dead ranges up to the next
265 // marked obj or hr_top.
266 HeapWord* next_marked_obj_addr = bitmap->get_next_marked_addr(obj_end_addr, hr_top);
267 garbage_words += zap_dead_objects(hr, obj_end_addr, next_marked_obj_addr);
268 obj_addr = next_marked_obj_addr;
269 } while (obj_addr < chunk_end);
270
271 assert(marked_words > 0 && num_marked_objs > 0, "inv");
272
273 stat.register_objects_count_and_size(num_marked_objs, marked_words);
274
275 update_garbage_words_in_hr(hr, garbage_words);
276 }
277
278 public:
279 RestoreEvacFailureRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
280 G1AbstractSubTask(G1GCPhaseTimes::RestoreEvacuationFailedRegions),
281 _g1h(G1CollectedHeap::heap()),
282 _cm(_g1h->concurrent_mark()),
283 _evac_failure_regions(evac_failure_regions),
284 _chunk_bitmap(mtGC) {
285
286 _num_evac_fail_regions = _evac_failure_regions->num_regions_evac_failed();
287 _num_chunks_per_region = G1CollectedHeap::get_chunks_per_region();
288
289 _chunk_size = static_cast<uint>(G1HeapRegion::GrainWords / _num_chunks_per_region);
290
291 log_debug(gc, ergo)("Initializing removing self forwards with %u chunks per region",
292 _num_chunks_per_region);
293
294 _chunk_bitmap.resize(_num_chunks_per_region * _num_evac_fail_regions);
295 }
296
297 double worker_cost() const override {
298 assert(_evac_failure_regions->has_regions_evac_failed(), "Should not call this if there were no evacuation failures");
299
300 double workers_per_region = (double)G1CollectedHeap::get_chunks_per_region() / G1RestoreRetainedRegionChunksPerWorker;
301 return workers_per_region * _evac_failure_regions->num_regions_evac_failed();
302 }
303
304 void do_work(uint worker_id) override {
305 const uint total_workers = G1CollectedHeap::heap()->workers()->active_workers();
306 const uint total_chunks = _num_chunks_per_region * _num_evac_fail_regions;
307 const uint start_chunk_idx = worker_id * total_chunks / total_workers;
308
309 for (uint i = 0; i < total_chunks; i++) {
310 const uint chunk_idx = (start_chunk_idx + i) % total_chunks;
311 if (claim_chunk(chunk_idx)) {
312 process_chunk(worker_id, chunk_idx);
313 }
314 }
315 }
316 };
317
318 G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
319 G1EvacFailureRegions* evac_failure_regions) :
320 G1BatchedTask("Post Evacuate Cleanup 1", G1CollectedHeap::heap()->phase_times())
321 {
322 bool evac_failed = evac_failure_regions->has_regions_evac_failed();
323 bool alloc_failed = evac_failure_regions->has_regions_alloc_failed();
324
325 add_serial_task(new MergePssTask(per_thread_states));
326 add_serial_task(new RecalculateUsedTask(evac_failed, alloc_failed));
327 if (SampleCollectionSetCandidatesTask::should_execute()) {
328 add_serial_task(new SampleCollectionSetCandidatesTask());
329 }
330 add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
331 if (evac_failed) {
332 add_parallel_task(new RestoreEvacFailureRegionsTask(evac_failure_regions));
333 }
334 }
335
336 class G1FreeHumongousRegionClosure : public HeapRegionIndexClosure {
337 uint _humongous_objects_reclaimed;
338 uint _humongous_regions_reclaimed;
339 size_t _freed_bytes;
340 G1CollectedHeap* _g1h;
341
342 // Returns whether the given humongous object defined by the start region index
343 // is reclaimable.
344 //
345 // At this point in the garbage collection, checking whether the humongous object
346 // is still a candidate is sufficient because:
347 //
348 // - if it has not been a candidate at the start of collection, it will never
349 // changed to be a candidate during the gc (and live).
350 // - any found outstanding (i.e. in the DCQ, or in its remembered set)
351 // references will set the candidate state to false.
352 // - there can be no references from within humongous starts regions referencing
353 // the object because we never allocate other objects into them.
354 // (I.e. there can be no intra-region references)
355 //
356 // It is not required to check whether the object has been found dead by marking
357 // or not, in fact it would prevent reclamation within a concurrent cycle, as
358 // all objects allocated during that time are considered live.
359 // SATB marking is even more conservative than the remembered set.
360 // So if at this point in the collection we did not find a reference during gc
361 // (or it had enough references to not be a candidate, having many remembered
362 // set entries), nobody has a reference to it.
363 // At the start of collection we flush all refinement logs, and remembered sets
364 // are completely up-to-date wrt to references to the humongous object.
365 //
366 // So there is no need to re-check remembered set size of the humongous region.
367 //
368 // Other implementation considerations:
369 // - never consider object arrays at this time because they would pose
370 // considerable effort for cleaning up the remembered sets. This is
371 // required because stale remembered sets might reference locations that
372 // are currently allocated into.
373 bool is_reclaimable(uint region_idx) const {
374 return G1CollectedHeap::heap()->is_humongous_reclaim_candidate(region_idx);
375 }
376
377 public:
378 G1FreeHumongousRegionClosure() :
379 _humongous_objects_reclaimed(0),
380 _humongous_regions_reclaimed(0),
381 _freed_bytes(0),
382 _g1h(G1CollectedHeap::heap())
383 {}
384
385 bool do_heap_region_index(uint region_index) override {
386 if (!is_reclaimable(region_index)) {
387 return false;
388 }
389
390 G1HeapRegion* r = _g1h->region_at(region_index);
391
392 oop obj = cast_to_oop(r->bottom());
393 guarantee(obj->is_typeArray(),
394 "Only eagerly reclaiming type arrays is supported, but the object "
395 PTR_FORMAT " is not.", p2i(r->bottom()));
396
397 log_debug(gc, humongous)("Reclaimed humongous region %u (object size " SIZE_FORMAT " @ " PTR_FORMAT ")",
398 region_index,
399 obj->size() * HeapWordSize,
400 p2i(r->bottom())
401 );
402
403 G1ConcurrentMark* const cm = _g1h->concurrent_mark();
404 cm->humongous_object_eagerly_reclaimed(r);
405 assert(!cm->is_marked_in_bitmap(obj),
406 "Eagerly reclaimed humongous region %u should not be marked at all but is in bitmap %s",
407 region_index,
408 BOOL_TO_STR(cm->is_marked_in_bitmap(obj)));
409 _humongous_objects_reclaimed++;
410
411 auto free_humongous_region = [&] (G1HeapRegion* r) {
412 _freed_bytes += r->used();
413 r->set_containing_set(nullptr);
414 _humongous_regions_reclaimed++;
415 G1HeapRegionPrinter::eager_reclaim(r);
416 _g1h->free_humongous_region(r, nullptr);
417 };
418
419 _g1h->humongous_obj_regions_iterate(r, free_humongous_region);
420
421 return false;
422 }
423
424 uint humongous_objects_reclaimed() {
425 return _humongous_objects_reclaimed;
426 }
427
428 uint humongous_regions_reclaimed() {
429 return _humongous_regions_reclaimed;
430 }
431
432 size_t bytes_freed() const {
433 return _freed_bytes;
434 }
435 };
436
437 #if COMPILER2_OR_JVMCI
438 class G1PostEvacuateCollectionSetCleanupTask2::UpdateDerivedPointersTask : public G1AbstractSubTask {
439 public:
440 UpdateDerivedPointersTask() : G1AbstractSubTask(G1GCPhaseTimes::UpdateDerivedPointers) { }
441
442 double worker_cost() const override { return 1.0; }
443 void do_work(uint worker_id) override { DerivedPointerTable::update_pointers(); }
444 };
445 #endif
446
447 class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask : public G1AbstractSubTask {
448 uint _humongous_regions_reclaimed;
449 size_t _bytes_freed;
450
451 public:
452 EagerlyReclaimHumongousObjectsTask() :
453 G1AbstractSubTask(G1GCPhaseTimes::EagerlyReclaimHumongousObjects),
454 _humongous_regions_reclaimed(0),
455 _bytes_freed(0) { }
456
457 virtual ~EagerlyReclaimHumongousObjectsTask() {
458 G1CollectedHeap* g1h = G1CollectedHeap::heap();
459
460 g1h->remove_from_old_gen_sets(0, _humongous_regions_reclaimed);
461 g1h->decrement_summary_bytes(_bytes_freed);
462 }
463
464 double worker_cost() const override { return 1.0; }
465 void do_work(uint worker_id) override {
466 G1CollectedHeap* g1h = G1CollectedHeap::heap();
467
468 G1FreeHumongousRegionClosure cl;
469 g1h->heap_region_iterate(&cl);
470
471 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumTotal, g1h->num_humongous_objects());
472 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumCandidates, g1h->num_humongous_reclaim_candidates());
473 record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumReclaimed, cl.humongous_objects_reclaimed());
474
475 _humongous_regions_reclaimed = cl.humongous_regions_reclaimed();
476 _bytes_freed = cl.bytes_freed();
477 }
478 };
479
480 class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask {
481 PreservedMarksSet* _preserved_marks;
482 WorkerTask* _task;
483
484 public:
485 RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) :
486 G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks),
487 _preserved_marks(preserved_marks),
488 _task(preserved_marks->create_task()) { }
489
490 virtual ~RestorePreservedMarksTask() {
491 delete _task;
492 }
493
494 double worker_cost() const override {
495 return _preserved_marks->num();
496 }
497
498 void do_work(uint worker_id) override { _task->work(worker_id); }
499 };
500
501 class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
502 size_t _num_dirtied;
503 G1CollectedHeap* _g1h;
504 G1CardTable* _g1_ct;
505 G1EvacFailureRegions* _evac_failure_regions;
506
507 G1HeapRegion* region_for_card(CardValue* card_ptr) const {
508 return _g1h->heap_region_containing(_g1_ct->addr_for(card_ptr));
509 }
510
511 bool will_become_free(G1HeapRegion* hr) const {
512 // A region will be freed by during the FreeCollectionSet phase if the region is in the
513 // collection set and has not had an evacuation failure.
514 return _g1h->is_in_cset(hr) && !_evac_failure_regions->contains(hr->hrm_index());
515 }
516
517 public:
518 RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h, G1EvacFailureRegions* evac_failure_regions) :
519 G1CardTableEntryClosure(),
520 _num_dirtied(0),
521 _g1h(g1h),
522 _g1_ct(g1h->card_table()),
523 _evac_failure_regions(evac_failure_regions) { }
524
525 void do_card_ptr(CardValue* card_ptr, uint worker_id) {
526 G1HeapRegion* hr = region_for_card(card_ptr);
527
528 // Should only dirty cards in regions that won't be freed.
529 if (!will_become_free(hr)) {
530 *card_ptr = G1CardTable::dirty_card_val();
531 _num_dirtied++;
532 }
533 }
534
535 size_t num_dirtied() const { return _num_dirtied; }
536 };
537
538 class G1PostEvacuateCollectionSetCleanupTask2::ProcessEvacuationFailedRegionsTask : public G1AbstractSubTask {
539 G1EvacFailureRegions* _evac_failure_regions;
540 HeapRegionClaimer _claimer;
541
542 class ProcessEvacuationFailedRegionsClosure : public HeapRegionClosure {
543 public:
544
545 bool do_heap_region(G1HeapRegion* r) override {
546 G1CollectedHeap* g1h = G1CollectedHeap::heap();
547 G1ConcurrentMark* cm = g1h->concurrent_mark();
548
549 HeapWord* top_at_mark_start = cm->top_at_mark_start(r);
550 assert(top_at_mark_start == r->bottom(), "TAMS must not have been set for region %u", r->hrm_index());
551 assert(cm->live_bytes(r->hrm_index()) == 0, "Marking live bytes must not be set for region %u", r->hrm_index());
552
553 // Concurrent mark does not mark through regions that we retain (they are root
554 // regions wrt to marking), so we must clear their mark data (tams, bitmap, ...)
555 // set eagerly or during evacuation failure.
556 bool clear_mark_data = !g1h->collector_state()->in_concurrent_start_gc() ||
557 g1h->policy()->should_retain_evac_failed_region(r);
558
559 if (clear_mark_data) {
560 g1h->clear_bitmap_for_region(r);
561 } else {
562 // This evacuation failed region is going to be marked through. Update mark data.
563 cm->update_top_at_mark_start(r);
564 cm->set_live_bytes(r->hrm_index(), r->live_bytes());
565 assert(cm->mark_bitmap()->get_next_marked_addr(r->bottom(), cm->top_at_mark_start(r)) != cm->top_at_mark_start(r),
566 "Marks must be on bitmap for region %u", r->hrm_index());
567 }
568 return false;
569 }
570 };
571
572 public:
573 ProcessEvacuationFailedRegionsTask(G1EvacFailureRegions* evac_failure_regions) :
574 G1AbstractSubTask(G1GCPhaseTimes::ProcessEvacuationFailedRegions),
575 _evac_failure_regions(evac_failure_regions),
576 _claimer(0) {
577 }
578
579 void set_max_workers(uint max_workers) override {
580 _claimer.set_n_workers(max_workers);
581 }
582
583 double worker_cost() const override {
584 return _evac_failure_regions->num_regions_evac_failed();
585 }
586
587 void do_work(uint worker_id) override {
588 ProcessEvacuationFailedRegionsClosure cl;
589 _evac_failure_regions->par_iterate(&cl, &_claimer, worker_id);
590 }
591 };
592
593 class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G1AbstractSubTask {
594 BufferNodeList* _rdc_buffers;
595 uint _num_buffer_lists;
596 G1EvacFailureRegions* _evac_failure_regions;
597
598 public:
599 RedirtyLoggedCardsTask(G1EvacFailureRegions* evac_failure_regions, BufferNodeList* rdc_buffers, uint num_buffer_lists) :
600 G1AbstractSubTask(G1GCPhaseTimes::RedirtyCards),
601 _rdc_buffers(rdc_buffers),
602 _num_buffer_lists(num_buffer_lists),
603 _evac_failure_regions(evac_failure_regions) { }
604
605 double worker_cost() const override {
606 // Needs more investigation.
607 return G1CollectedHeap::heap()->workers()->active_workers();
608 }
609
610 void do_work(uint worker_id) override {
611 RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap(), _evac_failure_regions);
612
613 uint start = worker_id;
614 for (uint i = 0; i < _num_buffer_lists; i++) {
615 uint index = (start + i) % _num_buffer_lists;
616
617 BufferNode* next = Atomic::load(&_rdc_buffers[index]._head);
618 BufferNode* tail = Atomic::load(&_rdc_buffers[index]._tail);
619
620 while (next != nullptr) {
621 BufferNode* node = next;
622 next = Atomic::cmpxchg(&_rdc_buffers[index]._head, node, (node != tail ) ? node->next() : nullptr);
623 if (next == node) {
624 cl.apply_to_buffer(node, worker_id);
625 next = (node != tail ) ? node->next() : nullptr;
626 } else {
627 break; // If there is contention, move to the next BufferNodeList
628 }
629 }
630 }
631 record_work_item(worker_id, 0, cl.num_dirtied());
632 }
633 };
634
635 // Helper class to keep statistics for the collection set freeing
636 class FreeCSetStats {
637 size_t _before_used_bytes; // Usage in regions successfully evacuate
638 size_t _after_used_bytes; // Usage in regions failing evacuation
639 size_t _bytes_allocated_in_old_since_last_gc; // Size of young regions turned into old
640 size_t _failure_used_words; // Live size in failed regions
641 size_t _failure_waste_words; // Wasted size in failed regions
642 size_t _card_rs_length; // (Card Set) Remembered set size
643 uint _regions_freed; // Number of regions freed
644
645 public:
646 FreeCSetStats() :
647 _before_used_bytes(0),
648 _after_used_bytes(0),
649 _bytes_allocated_in_old_since_last_gc(0),
650 _failure_used_words(0),
651 _failure_waste_words(0),
652 _card_rs_length(0),
653 _regions_freed(0) { }
654
655 void merge_stats(FreeCSetStats* other) {
656 assert(other != nullptr, "invariant");
657 _before_used_bytes += other->_before_used_bytes;
658 _after_used_bytes += other->_after_used_bytes;
659 _bytes_allocated_in_old_since_last_gc += other->_bytes_allocated_in_old_since_last_gc;
660 _failure_used_words += other->_failure_used_words;
661 _failure_waste_words += other->_failure_waste_words;
662 _card_rs_length += other->_card_rs_length;
663 _regions_freed += other->_regions_freed;
664 }
665
666 void report(G1CollectedHeap* g1h, G1EvacInfo* evacuation_info) {
667 evacuation_info->set_regions_freed(_regions_freed);
668 evacuation_info->set_collection_set_used_before(_before_used_bytes + _after_used_bytes);
669 evacuation_info->increment_collection_set_used_after(_after_used_bytes);
670
671 g1h->decrement_summary_bytes(_before_used_bytes);
672 g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->add_failure_used_and_waste(_failure_used_words, _failure_waste_words);
673
674 G1Policy *policy = g1h->policy();
675 policy->old_gen_alloc_tracker()->add_allocated_bytes_since_last_gc(_bytes_allocated_in_old_since_last_gc);
676 policy->record_card_rs_length(_card_rs_length);
677 policy->cset_regions_freed();
678 }
679
680 void account_failed_region(G1HeapRegion* r) {
681 size_t used_words = r->live_bytes() / HeapWordSize;
682 _failure_used_words += used_words;
683 _failure_waste_words += G1HeapRegion::GrainWords - used_words;
684 _after_used_bytes += r->used();
685
686 // When moving a young gen region to old gen, we "allocate" that whole
687 // region there. This is in addition to any already evacuated objects.
688 // Notify the policy about that. Old gen regions do not cause an
689 // additional allocation: both the objects still in the region and the
690 // ones already moved are accounted for elsewhere.
691 if (r->is_young()) {
692 _bytes_allocated_in_old_since_last_gc += G1HeapRegion::GrainBytes;
693 }
694 }
695
696 void account_evacuated_region(G1HeapRegion* r) {
697 size_t used = r->used();
698 assert(used > 0, "region %u %s zero used", r->hrm_index(), r->get_short_type_str());
699 _before_used_bytes += used;
700 _regions_freed += 1;
701 }
702
703 void account_card_rs_length(G1HeapRegion* r) {
704 _card_rs_length += r->rem_set()->occupied();
705 }
706 };
707
708 // Closure applied to all regions in the collection set.
709 class FreeCSetClosure : public HeapRegionClosure {
710 // Helper to send JFR events for regions.
711 class JFREventForRegion {
712 EventGCPhaseParallel _event;
713
714 public:
715 JFREventForRegion(G1HeapRegion* region, uint worker_id) : _event() {
716 _event.set_gcId(GCId::current());
717 _event.set_gcWorkerId(worker_id);
718 if (region->is_young()) {
719 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::YoungFreeCSet));
720 } else {
721 _event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::NonYoungFreeCSet));
722 }
723 }
724
725 ~JFREventForRegion() {
726 _event.commit();
727 }
728 };
729
730 // Helper to do timing for region work.
731 class TimerForRegion {
732 Tickspan& _time;
733 Ticks _start_time;
734 public:
735 TimerForRegion(Tickspan& time) : _time(time), _start_time(Ticks::now()) { }
736 ~TimerForRegion() {
737 _time += Ticks::now() - _start_time;
738 }
739 };
740
741 // FreeCSetClosure members
742 G1CollectedHeap* _g1h;
743 const size_t* _surviving_young_words;
744 uint _worker_id;
745 Tickspan _young_time;
746 Tickspan _non_young_time;
747 FreeCSetStats* _stats;
748 G1EvacFailureRegions* _evac_failure_regions;
749 uint _num_retained_regions;
750
751 void assert_tracks_surviving_words(G1HeapRegion* r) {
752 assert(r->young_index_in_cset() != 0 &&
753 (uint)r->young_index_in_cset() <= _g1h->collection_set()->young_region_length(),
754 "Young index %u is wrong for region %u of type %s with %u young regions",
755 r->young_index_in_cset(), r->hrm_index(), r->get_type_str(), _g1h->collection_set()->young_region_length());
756 }
757
758 void handle_evacuated_region(G1HeapRegion* r) {
759 assert(!r->is_empty(), "Region %u is an empty region in the collection set.", r->hrm_index());
760 stats()->account_evacuated_region(r);
761
762 G1HeapRegionPrinter::evac_reclaim(r);
763 // Free the region and its remembered set.
764 _g1h->free_region(r, nullptr);
765 }
766
767 void handle_failed_region(G1HeapRegion* r) {
768 // Do some allocation statistics accounting. Regions that failed evacuation
769 // are always made old, so there is no need to update anything in the young
770 // gen statistics, but we need to update old gen statistics.
771 stats()->account_failed_region(r);
772
773 G1GCPhaseTimes* p = _g1h->phase_times();
774 assert(r->in_collection_set(), "Failed evacuation of region %u not in collection set", r->hrm_index());
775
776 p->record_or_add_thread_work_item(G1GCPhaseTimes::RestoreEvacuationFailedRegions,
777 _worker_id,
778 1,
779 G1GCPhaseTimes::RestoreEvacFailureRegionsEvacFailedNum);
780
781 bool retain_region = _g1h->policy()->should_retain_evac_failed_region(r);
782 // Update the region state due to the failed evacuation.
783 r->handle_evacuation_failure(retain_region);
784 assert(r->is_old(), "must already be relabelled as old");
785
786 if (retain_region) {
787 _g1h->retain_region(r);
788 _num_retained_regions++;
789 }
790 assert(retain_region == r->rem_set()->is_tracked(), "When retaining a region, remembered set should be kept.");
791
792 // Add region to old set, need to hold lock.
793 MutexLocker x(OldSets_lock, Mutex::_no_safepoint_check_flag);
794 _g1h->old_set_add(r);
795 }
796
797 Tickspan& timer_for_region(G1HeapRegion* r) {
798 return r->is_young() ? _young_time : _non_young_time;
799 }
800
801 FreeCSetStats* stats() {
802 return _stats;
803 }
804
805 public:
806 FreeCSetClosure(const size_t* surviving_young_words,
807 uint worker_id,
808 FreeCSetStats* stats,
809 G1EvacFailureRegions* evac_failure_regions) :
810 HeapRegionClosure(),
811 _g1h(G1CollectedHeap::heap()),
812 _surviving_young_words(surviving_young_words),
813 _worker_id(worker_id),
814 _young_time(),
815 _non_young_time(),
816 _stats(stats),
817 _evac_failure_regions(evac_failure_regions),
818 _num_retained_regions(0) { }
819
820 virtual bool do_heap_region(G1HeapRegion* r) {
821 assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
822 JFREventForRegion event(r, _worker_id);
823 TimerForRegion timer(timer_for_region(r));
824
825 stats()->account_card_rs_length(r);
826
827 if (r->is_young()) {
828 assert_tracks_surviving_words(r);
829 r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
830 }
831
832 if (_evac_failure_regions->contains(r->hrm_index())) {
833 handle_failed_region(r);
834 } else {
835 handle_evacuated_region(r);
836 }
837 assert(!_g1h->is_on_master_free_list(r), "sanity");
838
839 return false;
840 }
841
842 void report_timing() {
843 G1GCPhaseTimes* pt = _g1h->phase_times();
844 if (_young_time.value() > 0) {
845 pt->record_time_secs(G1GCPhaseTimes::YoungFreeCSet, _worker_id, _young_time.seconds());
846 }
847 if (_non_young_time.value() > 0) {
848 pt->record_time_secs(G1GCPhaseTimes::NonYoungFreeCSet, _worker_id, _non_young_time.seconds());
849 }
850 }
851
852 bool num_retained_regions() const { return _num_retained_regions; }
853 };
854
855 class G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask : public G1AbstractSubTask {
856 G1CollectedHeap* _g1h;
857 G1EvacInfo* _evacuation_info;
858 FreeCSetStats* _worker_stats;
859 HeapRegionClaimer _claimer;
860 const size_t* _surviving_young_words;
861 uint _active_workers;
862 G1EvacFailureRegions* _evac_failure_regions;
863 volatile uint _num_retained_regions;
864
865 FreeCSetStats* worker_stats(uint worker) {
866 return &_worker_stats[worker];
867 }
868
869 void report_statistics() {
870 // Merge the accounting
871 FreeCSetStats total_stats;
872 for (uint worker = 0; worker < _active_workers; worker++) {
873 total_stats.merge_stats(worker_stats(worker));
874 }
875 total_stats.report(_g1h, _evacuation_info);
876 }
877
878 public:
879 FreeCollectionSetTask(G1EvacInfo* evacuation_info,
880 const size_t* surviving_young_words,
881 G1EvacFailureRegions* evac_failure_regions) :
882 G1AbstractSubTask(G1GCPhaseTimes::FreeCollectionSet),
883 _g1h(G1CollectedHeap::heap()),
884 _evacuation_info(evacuation_info),
885 _worker_stats(nullptr),
886 _claimer(0),
887 _surviving_young_words(surviving_young_words),
888 _active_workers(0),
889 _evac_failure_regions(evac_failure_regions),
890 _num_retained_regions(0) {
891
892 _g1h->clear_eden();
893 }
894
895 virtual ~FreeCollectionSetTask() {
896 Ticks serial_time = Ticks::now();
897
898 bool has_new_retained_regions = Atomic::load(&_num_retained_regions) != 0;
899 if (has_new_retained_regions) {
900 G1CollectionSetCandidates* candidates = _g1h->collection_set()->candidates();
901 candidates->sort_by_efficiency();
902 }
903
904 report_statistics();
905 for (uint worker = 0; worker < _active_workers; worker++) {
906 _worker_stats[worker].~FreeCSetStats();
907 }
908 FREE_C_HEAP_ARRAY(FreeCSetStats, _worker_stats);
909
910 G1GCPhaseTimes* p = _g1h->phase_times();
911 p->record_serial_free_cset_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
912
913 _g1h->clear_collection_set();
914 }
915
916 double worker_cost() const override { return G1CollectedHeap::heap()->collection_set()->region_length(); }
917
918 void set_max_workers(uint max_workers) override {
919 _active_workers = max_workers;
920 _worker_stats = NEW_C_HEAP_ARRAY(FreeCSetStats, max_workers, mtGC);
921 for (uint worker = 0; worker < _active_workers; worker++) {
922 ::new (&_worker_stats[worker]) FreeCSetStats();
923 }
924 _claimer.set_n_workers(_active_workers);
925 }
926
927 void do_work(uint worker_id) override {
928 FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id), _evac_failure_regions);
929 _g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
930 // Report per-region type timings.
931 cl.report_timing();
932
933 Atomic::add(&_num_retained_regions, cl.num_retained_regions(), memory_order_relaxed);
934 }
935 };
936
937 class G1PostEvacuateCollectionSetCleanupTask2::ResizeTLABsTask : public G1AbstractSubTask {
938 G1JavaThreadsListClaimer _claimer;
939
940 // There is not much work per thread so the number of threads per worker is high.
941 static const uint ThreadsPerWorker = 250;
942
943 public:
944 ResizeTLABsTask() : G1AbstractSubTask(G1GCPhaseTimes::ResizeThreadLABs), _claimer(ThreadsPerWorker) { }
945
946 void do_work(uint worker_id) override {
947 class ResizeClosure : public ThreadClosure {
948 public:
949
950 void do_thread(Thread* thread) {
951 static_cast<JavaThread*>(thread)->tlab().resize();
952 }
953 } cl;
954 _claimer.apply(&cl);
955 }
956
957 double worker_cost() const override {
958 return (double)_claimer.length() / ThreadsPerWorker;
959 }
960 };
961
962 G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
963 G1EvacInfo* evacuation_info,
964 G1EvacFailureRegions* evac_failure_regions) :
965 G1BatchedTask("Post Evacuate Cleanup 2", G1CollectedHeap::heap()->phase_times())
966 {
967 #if COMPILER2_OR_JVMCI
968 add_serial_task(new UpdateDerivedPointersTask());
969 #endif
970 if (G1CollectedHeap::heap()->has_humongous_reclaim_candidates()) {
971 add_serial_task(new EagerlyReclaimHumongousObjectsTask());
972 }
973
974 if (evac_failure_regions->has_regions_evac_failed()) {
975 add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
976 add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
977 }
978 add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,
979 per_thread_states->rdc_buffers(),
980 per_thread_states->num_workers()));
981
982 if (UseTLAB && ResizeTLAB) {
983 add_parallel_task(new ResizeTLABsTask());
984 }
985 add_parallel_task(new FreeCollectionSetTask(evacuation_info,
986 per_thread_states->surviving_young_words(),
987 evac_failure_regions));
988 }