1 /*
2 * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/classLoaderData.hpp"
26 #include "classfile/classLoaderDataGraph.hpp"
27 #include "code/codeCache.hpp"
28 #include "cppstdlib/new.hpp"
29 #include "gc/g1/g1BarrierSet.hpp"
30 #include "gc/g1/g1BatchedTask.hpp"
31 #include "gc/g1/g1CardSetMemory.hpp"
32 #include "gc/g1/g1CardTableClaimTable.inline.hpp"
33 #include "gc/g1/g1CollectedHeap.inline.hpp"
34 #include "gc/g1/g1CollectorState.inline.hpp"
35 #include "gc/g1/g1ConcurrentMark.inline.hpp"
36 #include "gc/g1/g1ConcurrentMarkRemarkTasks.hpp"
37 #include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
38 #include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
39 #include "gc/g1/g1ConcurrentRefine.hpp"
40 #include "gc/g1/g1HeapRegion.inline.hpp"
41 #include "gc/g1/g1HeapRegionManager.hpp"
42 #include "gc/g1/g1HeapRegionPrinter.hpp"
43 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
44 #include "gc/g1/g1HeapRegionSet.inline.hpp"
45 #include "gc/g1/g1HeapVerifier.hpp"
46 #include "gc/g1/g1OopClosures.inline.hpp"
47 #include "gc/g1/g1Policy.hpp"
48 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
49 #include "gc/g1/g1ThreadLocalData.hpp"
50 #include "gc/g1/g1Trace.hpp"
51 #include "gc/shared/gcId.hpp"
52 #include "gc/shared/gcTimer.hpp"
53 #include "gc/shared/gcTraceTime.inline.hpp"
54 #include "gc/shared/gcVMOperations.hpp"
55 #include "gc/shared/partialArraySplitter.inline.hpp"
56 #include "gc/shared/partialArrayState.hpp"
57 #include "gc/shared/partialArrayTaskStats.hpp"
58 #include "gc/shared/referencePolicy.hpp"
59 #include "gc/shared/suspendibleThreadSet.hpp"
60 #include "gc/shared/taskqueue.inline.hpp"
61 #include "gc/shared/taskTerminator.hpp"
62 #include "gc/shared/weakProcessor.inline.hpp"
63 #include "gc/shared/workerPolicy.hpp"
64 #include "jvm.h"
65 #include "logging/log.hpp"
66 #include "memory/allocation.hpp"
67 #include "memory/iterator.hpp"
68 #include "memory/metaspaceUtils.hpp"
69 #include "memory/resourceArea.hpp"
70 #include "memory/universe.hpp"
71 #include "nmt/memTracker.hpp"
72 #include "oops/access.inline.hpp"
73 #include "oops/oop.inline.hpp"
74 #include "oops/oopCast.inline.hpp"
75 #include "runtime/globals_extension.hpp"
76 #include "runtime/handles.inline.hpp"
77 #include "runtime/java.hpp"
78 #include "runtime/orderAccess.hpp"
79 #include "runtime/os.hpp"
80 #include "runtime/prefetch.inline.hpp"
81 #include "runtime/threads.hpp"
82 #include "utilities/align.hpp"
83 #include "utilities/checkedCast.hpp"
84 #include "utilities/formatBuffer.hpp"
85 #include "utilities/growableArray.hpp"
86 #include "utilities/powerOfTwo.hpp"
87
88 G1CMIsAliveClosure::G1CMIsAliveClosure() : _cm(nullptr) { }
89
90 G1CMIsAliveClosure::G1CMIsAliveClosure(G1ConcurrentMark* cm) : _cm(cm) {
91 assert(cm != nullptr, "must be");
92 }
93
94 void G1CMIsAliveClosure::initialize(G1ConcurrentMark* cm) {
95 assert(cm != nullptr, "must be");
96 assert(_cm == nullptr, "double initialize");
97 _cm = cm;
98 }
99
100 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
101 assert(addr < _cm->finger(), "invariant");
102 assert(addr >= _task->finger(), "invariant");
103
104 // We move that task's local finger along.
105 _task->move_finger_to(addr);
106
107 _task->process_entry(G1TaskQueueEntry(cast_to_oop(addr)), false /* stolen */);
108 // we only partially drain the local queue and global stack
109 _task->drain_local_queue(true);
110 _task->drain_global_stack(true);
111
112 // if the has_aborted flag has been raised, we need to bail out of
113 // the iteration
114 return !_task->has_aborted();
115 }
116
117 G1CMMarkStack::G1CMMarkStack() :
118 _chunk_allocator() {
119 set_empty();
120 }
121
122 size_t G1CMMarkStack::capacity_alignment() {
123 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
124 }
125
126 bool G1CMMarkStack::initialize() {
127 guarantee(_chunk_allocator.capacity() == 0, "G1CMMarkStack already initialized.");
128
129 size_t initial_capacity = MarkStackSize;
130 size_t max_capacity = MarkStackSizeMax;
131
132 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
133
134 size_t max_num_chunks = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
135 size_t initial_num_chunks = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
136
137 initial_num_chunks = round_up_power_of_2(initial_num_chunks);
138 max_num_chunks = MAX2(initial_num_chunks, max_num_chunks);
139
140 size_t limit = (INT_MAX - 1);
141 max_capacity = MIN2((max_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
142 initial_capacity = MIN2((initial_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
143
144 FLAG_SET_ERGO(MarkStackSizeMax, max_capacity);
145 FLAG_SET_ERGO(MarkStackSize, initial_capacity);
146
147 log_trace(gc)("MarkStackSize: %uk MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
148
149 log_debug(gc)("Initialize mark stack with %zu chunks, maximum %zu",
150 initial_num_chunks, max_capacity);
151
152 return _chunk_allocator.initialize(initial_num_chunks, max_num_chunks);
153 }
154
155 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_chunk() {
156 if (_size.load_relaxed() >= _max_capacity) {
157 return nullptr;
158 }
159
160 size_t cur_idx = _size.fetch_then_add(1u);
161
162 if (cur_idx >= _max_capacity) {
163 return nullptr;
164 }
165
166 size_t bucket = get_bucket(cur_idx);
167 if (_buckets[bucket].load_acquire() == nullptr) {
168 if (!_should_grow) {
169 // Prefer to restart the CM.
170 return nullptr;
171 }
172
173 MutexLocker x(G1MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
174 if (_buckets[bucket].load_acquire() == nullptr) {
175 size_t desired_capacity = bucket_size(bucket) * 2;
176 if (!try_expand_to(desired_capacity)) {
177 return nullptr;
178 }
179 }
180 }
181
182 size_t bucket_idx = get_bucket_index(cur_idx);
183 TaskQueueEntryChunk* result = ::new (&_buckets[bucket].load_relaxed()[bucket_idx]) TaskQueueEntryChunk;
184 result->next = nullptr;
185 return result;
186 }
187
188 G1CMMarkStack::ChunkAllocator::ChunkAllocator() :
189 _min_capacity(0),
190 _max_capacity(0),
191 _capacity(0),
192 _num_buckets(0),
193 _should_grow(false),
194 _buckets(nullptr),
195 _size(0)
196 { }
197
198 bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t max_capacity) {
199 guarantee(is_power_of_2(initial_capacity), "Invalid initial_capacity");
200
201 _min_capacity = initial_capacity;
202 _max_capacity = max_capacity;
203 _num_buckets = get_bucket(_max_capacity) + 1;
204
205 _buckets = NEW_C_HEAP_ARRAY(Atomic<TaskQueueEntryChunk*>, _num_buckets, mtGC);
206
207 for (size_t i = 0; i < _num_buckets; i++) {
208 _buckets[i].store_relaxed(nullptr);
209 }
210
211 size_t new_capacity = bucket_size(0);
212
213 if (!reserve(new_capacity)) {
214 log_warning(gc)("Failed to reserve memory for new overflow mark stack with %zu chunks and size %zuB.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
215 return false;
216 }
217 return true;
218 }
219
220 bool G1CMMarkStack::ChunkAllocator::try_expand_to(size_t desired_capacity) {
221 if (_capacity == _max_capacity) {
222 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of %zu chunks.", _capacity);
223 return false;
224 }
225
226 size_t old_capacity = _capacity;
227 desired_capacity = MIN2(desired_capacity, _max_capacity);
228
229 if (reserve(desired_capacity)) {
230 log_debug(gc)("Expanded the mark stack capacity from %zu to %zu chunks",
231 old_capacity, desired_capacity);
232 return true;
233 }
234 return false;
235 }
236
237 bool G1CMMarkStack::ChunkAllocator::try_expand() {
238 size_t new_capacity = _capacity * 2;
239 return try_expand_to(new_capacity);
240 }
241
242 G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
243 if (_buckets == nullptr) {
244 return;
245 }
246
247 for (size_t i = 0; i < _num_buckets; i++) {
248 if (_buckets[i].load_relaxed() != nullptr) {
249 MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i].load_relaxed(), bucket_size(i));
250 _buckets[i].store_relaxed(nullptr);
251 }
252 }
253
254 FREE_C_HEAP_ARRAY(_buckets);
255 }
256
257 bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
258 assert(new_capacity <= _max_capacity, "Cannot expand overflow mark stack beyond the max_capacity of %zu chunks.", _max_capacity);
259
260 size_t highest_bucket = get_bucket(new_capacity - 1);
261 size_t i = get_bucket(_capacity);
262
263 // Allocate all buckets associated with indexes between the current capacity (_capacity)
264 // and the new capacity (new_capacity). This step ensures that there are no gaps in the
265 // array and that the capacity accurately reflects the reserved memory.
266 for (; i <= highest_bucket; i++) {
267 if (_buckets[i].load_acquire() != nullptr) {
268 continue; // Skip over already allocated buckets.
269 }
270
271 size_t bucket_capacity = bucket_size(i);
272
273 // Trim bucket size so that we do not exceed the _max_capacity.
274 bucket_capacity = (_capacity + bucket_capacity) <= _max_capacity ?
275 bucket_capacity :
276 _max_capacity - _capacity;
277
278
279 TaskQueueEntryChunk* bucket_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(bucket_capacity, mtGC);
280
281 if (bucket_base == nullptr) {
282 log_warning(gc)("Failed to reserve memory for increasing the overflow mark stack capacity with %zu chunks and size %zuB.",
283 bucket_capacity, bucket_capacity * sizeof(TaskQueueEntryChunk));
284 return false;
285 }
286 _capacity += bucket_capacity;
287 _buckets[i].release_store(bucket_base);
288 }
289 return true;
290 }
291
292 void G1CMMarkStack::expand() {
293 _chunk_allocator.try_expand();
294 }
295
296 void G1CMMarkStack::add_chunk_to_list(Atomic<TaskQueueEntryChunk*>* list, TaskQueueEntryChunk* elem) {
297 elem->next = list->load_relaxed();
298 list->store_relaxed(elem);
299 }
300
301 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) {
302 MutexLocker x(G1MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
303 add_chunk_to_list(&_chunk_list, elem);
304 _chunks_in_chunk_list.add_then_fetch(1u, memory_order_relaxed);
305 }
306
307 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) {
308 MutexLocker x(G1MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
309 add_chunk_to_list(&_free_list, elem);
310 }
311
312 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(Atomic<TaskQueueEntryChunk*>* list) {
313 TaskQueueEntryChunk* result = list->load_relaxed();
314 if (result != nullptr) {
315 list->store_relaxed(list->load_relaxed()->next);
316 }
317 return result;
318 }
319
320 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() {
321 MutexLocker x(G1MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
322 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list);
323 if (result != nullptr) {
324 _chunks_in_chunk_list.sub_then_fetch(1u, memory_order_relaxed);
325 }
326 return result;
327 }
328
329 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() {
330 MutexLocker x(G1MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
331 return remove_chunk_from_list(&_free_list);
332 }
333
334 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
335 // Get a new chunk.
336 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
337
338 if (new_chunk == nullptr) {
339 // Did not get a chunk from the free list. Allocate from backing memory.
340 new_chunk = _chunk_allocator.allocate_new_chunk();
341
342 if (new_chunk == nullptr) {
343 return false;
344 }
345 }
346
347 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry));
348
349 add_chunk_to_chunk_list(new_chunk);
350
351 return true;
352 }
353
354 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
355 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list();
356
357 if (cur == nullptr) {
358 return false;
359 }
360
361 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry));
362
363 add_chunk_to_free_list(cur);
364 return true;
365 }
366
367 void G1CMMarkStack::set_empty() {
368 _chunks_in_chunk_list.store_relaxed(0);
369 _chunk_list.store_relaxed(nullptr);
370 _free_list.store_relaxed(nullptr);
371 _chunk_allocator.reset();
372 }
373
374 G1CMRootMemRegions::G1CMRootMemRegions(uint const max_regions) :
375 _root_regions(MemRegion::create_array(max_regions, mtGC)),
376 _max_regions(max_regions),
377 _num_regions(0),
378 _num_claimed_regions(0) { }
379
380 G1CMRootMemRegions::~G1CMRootMemRegions() {
381 MemRegion::destroy_array(_root_regions, _max_regions);
382 }
383
384 void G1CMRootMemRegions::reset() {
385 assert_at_safepoint();
386 assert(G1CollectedHeap::heap()->collector_state()->is_in_concurrent_start_gc(), "must be");
387
388 _num_regions.store_relaxed(0);
389 _num_claimed_regions.store_relaxed(0);
390 }
391
392 void G1CMRootMemRegions::add(HeapWord* start, HeapWord* end) {
393 assert_at_safepoint();
394 uint idx = _num_regions.fetch_then_add(1u);
395 assert(idx < _max_regions, "Trying to add more root MemRegions than there is space %u", _max_regions);
396 assert(start != nullptr && end != nullptr && start <= end, "Start (" PTR_FORMAT ") should be less or equal to "
397 "end (" PTR_FORMAT ")", p2i(start), p2i(end));
398 _root_regions[idx].set_start(start);
399 _root_regions[idx].set_end(end);
400 }
401
402 const MemRegion* G1CMRootMemRegions::claim_next() {
403 uint local_num_regions = num_regions();
404 if (num_claimed_regions() >= local_num_regions) {
405 return nullptr;
406 }
407
408 uint claimed_index = _num_claimed_regions.fetch_then_add(1u);
409 if (claimed_index < local_num_regions) {
410 return &_root_regions[claimed_index];
411 }
412 return nullptr;
413 }
414
415 bool G1CMRootMemRegions::work_completed() const {
416 return num_remaining_regions() == 0;
417 }
418
419 uint G1CMRootMemRegions::num_remaining_regions() const {
420 uint total = num_regions();
421 uint claimed = num_claimed_regions();
422 return (total > claimed) ? total - claimed : 0;
423 }
424
425 bool G1CMRootMemRegions::contains(const MemRegion mr) const {
426 uint local_num_root_regions = num_regions();
427 for (uint i = 0; i < local_num_root_regions; i++) {
428 if (_root_regions[i].equals(mr)) {
429 return true;
430 }
431 }
432 return false;
433 }
434
435 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
436 G1RegionToSpaceMapper* bitmap_storage) :
437 _cm_thread(nullptr),
438 _g1h(g1h),
439
440 _mark_bitmap(),
441
442 _heap(_g1h->reserved()),
443
444 _root_regions(_g1h->max_num_regions()),
445 _root_region_scan_aborted(false),
446
447 _global_mark_stack(),
448
449 _finger(nullptr), // _finger set in set_non_marking_state
450
451 _worker_id_offset(G1ConcRefinementThreads), // The refinement control thread does not refine cards, so it's just the worker threads.
452 _max_num_tasks(MAX2(ConcGCThreads, ParallelGCThreads)),
453 _num_active_tasks(0), // _num_active_tasks set in set_non_marking_state()
454 _tasks(nullptr),
455 _task_queues(new G1CMTaskQueueSet(_max_num_tasks)),
456 _terminator(_max_num_tasks, _task_queues),
457 _partial_array_state_manager(new PartialArrayStateManager(_max_num_tasks)),
458
459 _first_overflow_barrier_sync(),
460 _second_overflow_barrier_sync(),
461
462 _completed_mark_cycles(0),
463 _has_overflown(false),
464 _concurrent(false),
465 _has_aborted(false),
466 _restart_for_overflow(false),
467 _gc_timer_cm(new ConcurrentGCTimer()),
468 _gc_tracer_cm(new G1OldTracer()),
469
470 // _verbose_level set below
471
472 _remark_times(),
473 _remark_mark_times(),
474 _remark_weak_ref_times(),
475 _cleanup_times(),
476
477 _concurrent_workers(nullptr),
478 _num_concurrent_workers(0),
479 _max_concurrent_workers(0),
480
481 _is_region_mark_stats_cache_in_use(false),
482 _region_mark_stats(nullptr),
483 _top_at_mark_starts(nullptr),
484 _top_at_rebuild_starts(nullptr),
485 _needs_remembered_set_rebuild(false)
486 {
487 assert(G1CGC_lock != nullptr, "CGC_lock must be initialized");
488
489 _mark_bitmap.initialize(g1h->reserved(), bitmap_storage);
490 }
491
492 void G1ConcurrentMark::fully_initialize() {
493 assert_at_safepoint();
494
495 if (is_fully_initialized()) {
496 return;
497 }
498
499 // Create & start ConcurrentMark thread.
500 _cm_thread = new G1ConcurrentMarkThread(this);
501 if (_cm_thread->osthread() == nullptr) {
502 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
503 }
504
505 log_debug(gc)("ConcGCThreads: %u offset %u", ConcGCThreads, _worker_id_offset);
506 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
507
508 _max_concurrent_workers = ConcGCThreads;
509
510 _concurrent_workers = new WorkerThreads("G1 Conc", _max_concurrent_workers);
511 _concurrent_workers->initialize_workers();
512 _num_concurrent_workers = _concurrent_workers->active_workers();
513
514 if (!_global_mark_stack.initialize()) {
515 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
516 }
517
518 _region_mark_stats = NEW_C_HEAP_ARRAY(G1RegionMarkStats, _g1h->max_num_regions(), mtGC);
519 _top_at_mark_starts = NEW_C_HEAP_ARRAY(Atomic<HeapWord*>, _g1h->max_num_regions(), mtGC);
520 _top_at_rebuild_starts = NEW_C_HEAP_ARRAY(Atomic<HeapWord*>, _g1h->max_num_regions(), mtGC);
521
522 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC);
523
524 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
525 _num_active_tasks = _max_num_tasks;
526
527 for (uint i = 0; i < _max_num_tasks; ++i) {
528 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
529 _task_queues->register_queue(i, task_queue);
530
531 _tasks[i] = new G1CMTask(i, this, task_queue, _region_mark_stats);
532 }
533
534 uint max_num_regions = _g1h->max_num_regions();
535 ::new (_region_mark_stats) G1RegionMarkStats[max_num_regions]{};
536 for (uint i = 0; i < max_num_regions; i++) {
537 ::new (&_top_at_mark_starts[i]) Atomic<HeapWord*>(_g1h->bottom_addr_for_region(i));
538 }
539 ::new (_top_at_rebuild_starts) Atomic<HeapWord*>[max_num_regions]{};
540
541 reset_at_marking_complete();
542 }
543
544 bool G1ConcurrentMark::is_in_concurrent_cycle() const {
545 return _cm_thread->is_in_progress();
546 }
547
548 bool G1ConcurrentMark::is_in_marking() const {
549 return _cm_thread->is_in_marking();
550 }
551
552 bool G1ConcurrentMark::is_in_marking_or_rebuild() const {
553 return _cm_thread->is_in_marking_or_rebuild();
554 }
555
556 bool G1ConcurrentMark::is_in_reset_for_next_cycle() const {
557 return cm_thread()->is_in_reset_for_next_cycle();
558 }
559
560 PartialArrayStateManager* G1ConcurrentMark::partial_array_state_manager() const {
561 return _partial_array_state_manager;
562 }
563
564 G1ConcurrentMarkThread* G1ConcurrentMark::cm_thread() const {
565 assert(is_fully_initialized(), "must be");
566 return _cm_thread;
567 }
568
569 void G1ConcurrentMark::reset() {
570 assert_fully_initialized();
571
572 _has_aborted.store_relaxed(false);
573
574 _is_region_mark_stats_cache_in_use = true;
575 reset_marking_for_restart();
576
577 // Reset all tasks, since different phases will use different number of active
578 // threads. So, it's easiest to have all of them ready.
579 for (uint i = 0; i < _max_num_tasks; ++i) {
580 _tasks[i]->reset(mark_bitmap());
581 }
582
583 uint max_num_regions = _g1h->max_num_regions();
584 ::new (_top_at_rebuild_starts) Atomic<HeapWord*>[max_num_regions]{};
585 for (uint i = 0; i < max_num_regions; i++) {
586 // Do not update TAMS here. NoteStartOfMarkTask updates this in parallel in
587 // the pre-concurrent-start WorkerTask.
588 _top_at_rebuild_starts[i].store_relaxed(nullptr);
589 _region_mark_stats[i].clear();
590 }
591
592 _root_region_scan_aborted.store_relaxed(false);
593 _root_regions.reset();
594 }
595
596 void G1ConcurrentMark::assert_statistics_clear(G1HeapRegion* r) {
597 assert_fully_initialized();
598 #ifdef ASSERT
599 uint region_idx = r->hrm_index();
600 for (uint j = 0; j < _max_num_tasks; ++j) {
601 _tasks[j]->verify_no_mark_stats_for(r->hrm_index());
602 }
603
604 assert(_top_at_rebuild_starts[region_idx].load_relaxed() == nullptr, "must be");
605
606 G1RegionMarkStats* s = &_region_mark_stats[region_idx];
607 assert(s->incoming_refs() == 0, "must be");
608 assert(s->live_words() == 0, "must be");
609 #endif
610 }
611
612 void G1ConcurrentMark::note_start_of_mark_for_region(G1HeapRegion* r) {
613 assert_at_safepoint();
614 assert_fully_initialized();
615 if (r->is_old_or_humongous() && !r->is_collection_set_candidate() && !r->in_collection_set()) {
616 update_top_at_mark_start(r);
617 } else {
618 set_top_at_mark_start_to_bottom(r);
619 }
620 }
621
622 void G1ConcurrentMark::notify_new_region(G1HeapRegion* r, size_t marked_live_bytes_below_tams) {
623 assert_at_safepoint();
624 if (!is_fully_initialized()) {
625 return;
626 }
627 G1CollectorState* state = _g1h->collector_state();
628 if (state->is_in_concurrent_start_gc()) {
629 update_top_at_mark_start(r);
630 set_live_bytes(r->hrm_index(), marked_live_bytes_below_tams);
631 }
632 }
633
634 void G1ConcurrentMark::reset_region_marking_state(G1HeapRegion* r) {
635 assert_at_safepoint();
636 if (!is_fully_initialized()) {
637 return;
638 }
639 uint region_idx = r->hrm_index();
640 // Only need to clear the stats cache for the given region if we are using the cache.
641 if (_is_region_mark_stats_cache_in_use) {
642 for (uint j = 0; j < _max_num_tasks; ++j) {
643 _tasks[j]->clear_mark_stats_cache(region_idx);
644 }
645 } else {
646 for (uint j = 0; j < _max_num_tasks; ++j) {
647 _tasks[j]->verify_no_mark_stats_for(region_idx);
648 }
649 }
650 set_top_at_mark_start_to_bottom(r);
651 _top_at_rebuild_starts[region_idx].store_relaxed(nullptr);
652 _region_mark_stats[region_idx].clear();
653 }
654
655 void G1ConcurrentMark::humongous_object_eagerly_reclaimed(G1HeapRegion* r) {
656 assert_at_safepoint();
657 assert(r->is_starts_humongous(), "Got humongous continues region here");
658
659 // Need to clear mark bit of the humongous object. Doing this unconditionally is fine.
660 mark_bitmap()->clear(r->bottom());
661 }
662
663 void G1ConcurrentMark::reset_marking_for_restart() {
664 assert_fully_initialized();
665
666 _global_mark_stack.set_empty();
667
668 // Expand the marking stack, if we have to and if we can.
669 if (has_overflown()) {
670 _global_mark_stack.expand();
671
672 uint max_num_regions = _g1h->max_num_regions();
673 for (uint i = 0; i < max_num_regions; i++) {
674 _region_mark_stats[i].clear_during_overflow();
675 }
676 }
677
678 clear_has_overflown();
679 _finger.store_relaxed(_heap.start());
680
681 for (uint i = 0; i < _max_num_tasks; ++i) {
682 _tasks[i]->reset_for_restart();
683 }
684 }
685
686 void G1ConcurrentMark::set_concurrency(uint active_tasks) {
687 assert(active_tasks <= _max_num_tasks, "we should not have more");
688
689 _num_active_tasks = active_tasks;
690 // Need to update the three data structures below according to the
691 // number of active threads for this phase.
692 _terminator.reset_for_reuse(active_tasks);
693 _first_overflow_barrier_sync.set_n_workers(active_tasks);
694 _second_overflow_barrier_sync.set_n_workers(active_tasks);
695 }
696
697 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
698 set_concurrency(active_tasks);
699
700 _concurrent.store_relaxed(concurrent);
701
702 if (!concurrent) {
703 // At this point we should be in a STW phase, and completed marking.
704 assert_at_safepoint_on_vm_thread();
705 assert(out_of_regions(),
706 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT,
707 p2i(finger()), p2i(_heap.end()));
708 }
709 }
710
711 #if TASKQUEUE_STATS
712 void G1ConcurrentMark::print_and_reset_taskqueue_stats() {
713
714 _task_queues->print_and_reset_taskqueue_stats("Concurrent Mark");
715
716 auto get_pa_stats = [&](uint i) {
717 return _tasks[i]->partial_array_task_stats();
718 };
719
720 PartialArrayTaskStats::log_set(_max_num_tasks, get_pa_stats,
721 "Concurrent Mark Partial Array");
722
723 for (uint i = 0; i < _max_num_tasks; ++i) {
724 get_pa_stats(i)->reset();
725 }
726 }
727 #endif
728
729 void G1ConcurrentMark::reset_at_marking_complete() {
730 TASKQUEUE_STATS_ONLY(print_and_reset_taskqueue_stats());
731 // We set the global marking state to some default values when we're
732 // not doing marking.
733 reset_marking_for_restart();
734 _num_active_tasks = 0;
735 }
736
737 G1ConcurrentMark::~G1ConcurrentMark() {
738 FREE_C_HEAP_ARRAY(_top_at_mark_starts);
739 FREE_C_HEAP_ARRAY(_top_at_rebuild_starts);
740 FREE_C_HEAP_ARRAY(_region_mark_stats);
741 // The G1ConcurrentMark instance is never freed.
742 ShouldNotReachHere();
743 }
744
745 class G1ClearBitMapTask : public WorkerTask {
746 public:
747 static size_t chunk_size() { return M; }
748
749 private:
750 // Heap region closure used for clearing the _mark_bitmap.
751 class G1ClearBitmapHRClosure : public G1HeapRegionClosure {
752 G1ConcurrentMark* _cm;
753 G1CMBitMap* _bitmap;
754 bool _suspendible; // If suspendible, do yield checks.
755
756 bool suspendible() {
757 return _suspendible;
758 }
759
760 bool is_clear_concurrent_undo() {
761 return suspendible() && _cm->cm_thread()->is_in_undo_cycle();
762 }
763
764 bool has_aborted() {
765 if (suspendible()) {
766 _cm->do_yield_check();
767 return _cm->has_aborted();
768 }
769 return false;
770 }
771
772 HeapWord* region_clear_limit(G1HeapRegion* r) {
773 // A garbage collection might have made the region unavailable after a yield during
774 // clearing. Just return bottom as the limit, causing the clearing for this region to end.
775 if (G1CollectedHeap::heap()->region_at_or_null(r->hrm_index()) == nullptr) {
776 return r->bottom();
777 }
778 // During a Concurrent Undo Mark cycle, the per region top_at_mark_start and
779 // live_words data are current wrt to the _mark_bitmap. We use this information
780 // to only clear ranges of the bitmap that require clearing.
781 if (is_clear_concurrent_undo()) {
782 // No need to clear bitmaps for empty regions (which includes regions we
783 // did not mark through).
784 if (!_cm->contains_live_object(r->hrm_index())) {
785 assert(_bitmap->get_next_marked_addr(r->bottom(), r->end()) == r->end(), "Should not have marked bits");
786 return r->bottom();
787 }
788 }
789 return r->end();
790 }
791
792 public:
793 G1ClearBitmapHRClosure(G1ConcurrentMark* cm, bool suspendible) :
794 G1HeapRegionClosure(),
795 _cm(cm),
796 _bitmap(cm->mark_bitmap()),
797 _suspendible(suspendible)
798 { }
799
800 virtual bool do_heap_region(G1HeapRegion* r) {
801 if (has_aborted()) {
802 return true;
803 }
804
805 HeapWord* cur = r->bottom();
806 HeapWord* end = region_clear_limit(r);
807
808 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize;
809
810 while (cur < end) {
811
812 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end));
813 _bitmap->clear_range(mr);
814
815 cur += chunk_size_in_words;
816
817 // Repeat the asserts from before the start of the closure. We will do them
818 // as asserts here to minimize their overhead on the product. However, we
819 // will have them as guarantees at the beginning / end of the bitmap
820 // clearing to get some checking in the product.
821 assert(!suspendible() || _cm->is_in_reset_for_next_cycle(), "invariant");
822
823 // Abort iteration if necessary.
824 if (suspendible() && _cm->do_yield_check()) {
825 if (_cm->has_aborted()) {
826 return true;
827 }
828 // Re-read end. The region might have been uncommitted.
829 end = region_clear_limit(r);
830 }
831 }
832 assert(cur >= end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index());
833
834 return false;
835 }
836 };
837
838 G1ClearBitmapHRClosure _cl;
839 G1HeapRegionClaimer _hr_claimer;
840 bool _suspendible; // If the task is suspendible, workers must join the STS.
841
842 public:
843 G1ClearBitMapTask(G1ConcurrentMark* cm, uint n_workers, bool suspendible) :
844 WorkerTask("G1 Clear Bitmap"),
845 _cl(cm, suspendible),
846 _hr_claimer(n_workers),
847 _suspendible(suspendible)
848 { }
849
850 void work(uint worker_id) {
851 SuspendibleThreadSetJoiner sts_join(_suspendible);
852 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
853 }
854 };
855
856 void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers, bool may_yield) {
857 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
858
859 size_t const num_bytes_to_clear = (G1HeapRegion::GrainBytes * _g1h->num_committed_regions()) / G1CMBitMap::heap_map_factor();
860 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size();
861
862 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
863
864 G1ClearBitMapTask cl(this, num_workers, may_yield);
865
866 log_debug(gc, ergo)("Running %s with %u workers for %zu work units.", cl.name(), num_workers, num_chunks);
867 workers->run_task(&cl, num_workers);
868 }
869
870 void G1ConcurrentMark::cleanup_for_next_mark() {
871 // Make sure that the concurrent mark thread looks to still be in
872 // the current cycle.
873 guarantee(is_in_reset_for_next_cycle(), "invariant");
874
875 clear_bitmap(_concurrent_workers, true);
876
877 reset_partial_array_state_manager();
878
879 // Should not have changed state yet (even if a Full GC interrupted us).
880 guarantee(is_in_reset_for_next_cycle(), "invariant");
881 }
882
883 void G1ConcurrentMark::reset_partial_array_state_manager() {
884 for (uint i = 0; i < _max_num_tasks; ++i) {
885 _tasks[i]->unregister_partial_array_splitter();
886 }
887
888 partial_array_state_manager()->reset();
889
890 for (uint i = 0; i < _max_num_tasks; ++i) {
891 _tasks[i]->register_partial_array_splitter();
892 }
893 }
894
895 void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers) {
896 assert_at_safepoint_on_vm_thread();
897 // To avoid fragmentation the full collection requesting to clear the bitmap
898 // might use fewer workers than available. To ensure the bitmap is cleared
899 // as efficiently as possible the number of active workers are temporarily
900 // increased to include all currently created workers.
901 WithActiveWorkers update(workers, workers->created_workers());
902 clear_bitmap(workers, false);
903 }
904
905 class G1PreConcurrentStartTask : public G1BatchedTask {
906 // Reset marking state.
907 class ResetMarkingStateTask;
908 // For each region note start of marking.
909 class NoteStartOfMarkTask;
910
911 public:
912 G1PreConcurrentStartTask(GCCause::Cause cause, G1ConcurrentMark* cm);
913 };
914
915 class G1PreConcurrentStartTask::ResetMarkingStateTask : public G1AbstractSubTask {
916 G1ConcurrentMark* _cm;
917 public:
918 ResetMarkingStateTask(G1ConcurrentMark* cm) : G1AbstractSubTask(G1GCPhaseTimes::ResetMarkingState), _cm(cm) { }
919
920 double worker_cost() const override { return 1.0; }
921 void do_work(uint worker_id) override;
922 };
923
924 class G1PreConcurrentStartTask::NoteStartOfMarkTask : public G1AbstractSubTask {
925
926 class NoteStartOfMarkHRClosure : public G1HeapRegionClosure {
927 G1ConcurrentMark* _cm;
928
929 public:
930 NoteStartOfMarkHRClosure() : G1HeapRegionClosure(), _cm(G1CollectedHeap::heap()->concurrent_mark()) { }
931
932 bool do_heap_region(G1HeapRegion* r) override {
933 _cm->note_start_of_mark_for_region(r);
934 return false;
935 }
936 } _region_cl;
937
938 G1HeapRegionClaimer _claimer;
939 public:
940 NoteStartOfMarkTask() : G1AbstractSubTask(G1GCPhaseTimes::NoteStartOfMark), _region_cl(), _claimer(0) { }
941
942 double worker_cost() const override {
943 // The work done per region is very small, therefore we choose this magic number to cap the number
944 // of threads used when there are few regions.
945 const double regions_per_thread = 1000;
946 return _claimer.n_regions() / regions_per_thread;
947 }
948
949 void set_max_workers(uint max_workers) override {
950 _claimer.set_n_workers(max_workers);
951 }
952
953 void do_work(uint worker_id) override {
954 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_region_cl, &_claimer, worker_id);
955 }
956 };
957
958 void G1PreConcurrentStartTask::ResetMarkingStateTask::do_work(uint worker_id) {
959 // Reset marking state.
960 _cm->reset();
961 }
962
963 G1PreConcurrentStartTask::G1PreConcurrentStartTask(GCCause::Cause cause, G1ConcurrentMark* cm) :
964 G1BatchedTask("Pre Concurrent Start", G1CollectedHeap::heap()->phase_times()) {
965 add_serial_task(new ResetMarkingStateTask(cm));
966 add_parallel_task(new NoteStartOfMarkTask());
967 };
968
969 void G1ConcurrentMark::pre_concurrent_start(GCCause::Cause cause) {
970 assert_at_safepoint_on_vm_thread();
971
972 G1CollectedHeap::start_codecache_marking_cycle_if_inactive(true /* concurrent_mark_start */);
973
974 ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_strong);
975
976 G1PreConcurrentStartTask cl(cause, this);
977 G1CollectedHeap::heap()->run_batch_task(&cl);
978
979 _gc_tracer_cm->set_gc_cause(cause);
980 }
981
982 void G1ConcurrentMark::start_full_concurrent_cycle() {
983 // Start Concurrent Marking weak-reference discovery.
984 ReferenceProcessor* rp = _g1h->ref_processor_cm();
985 rp->start_discovery(false /* always_clear */);
986
987 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
988 // This is the start of the marking cycle, we're expected all
989 // threads to have SATB queues with active set to false.
990 satb_mq_set.set_active_all_threads(true, /* new active value */
991 false /* expected_active */);
992
993 // update_g1_committed() will be called at the end of an evac pause
994 // when marking is on. So, it's also called at the end of the
995 // concurrent start pause to update the heap end, if the heap expands
996 // during it. No need to call it here.
997
998 // Signal the thread to start work.
999 cm_thread()->start_full_cycle();
1000 }
1001
1002 void G1ConcurrentMark::start_undo_concurrent_cycle() {
1003 assert_at_safepoint_on_vm_thread();
1004 // At this time this GC is not a concurrent start gc any more, can only check for young only gc/phase.
1005 assert(_g1h->collector_state()->is_in_young_only_phase(), "must be");
1006
1007 abort_root_region_scan_at_safepoint();
1008
1009 // Signal the thread to start work.
1010 cm_thread()->start_undo_cycle();
1011 }
1012
1013 void G1ConcurrentMark::notify_concurrent_cycle_completed() {
1014 cm_thread()->set_idle();
1015 }
1016
1017 void G1ConcurrentMark::stop() {
1018 if (is_fully_initialized()) {
1019 cm_thread()->stop();
1020 }
1021 }
1022
1023 /*
1024 * Notice that in the next two methods, we actually leave the STS
1025 * during the barrier sync and join it immediately afterwards. If we
1026 * do not do this, the following deadlock can occur: one thread could
1027 * be in the barrier sync code, waiting for the other thread to also
1028 * sync up, whereas another one could be trying to yield, while also
1029 * waiting for the other threads to sync up too.
1030 *
1031 * Note, however, that this code is also used during remark and in
1032 * this case we should not attempt to leave / enter the STS, otherwise
1033 * we'll either hit an assert (debug / fastdebug) or deadlock
1034 * (product). So we should only leave / enter the STS if we are
1035 * operating concurrently.
1036 *
1037 * Because the thread that does the sync barrier has left the STS, it
1038 * is possible to be suspended for a Full GC or an evacuation pause
1039 * could occur. This is actually safe, since the entering the sync
1040 * barrier is one of the last things do_marking_step() does, and it
1041 * doesn't manipulate any data structures afterwards.
1042 */
1043
1044 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
1045 bool barrier_aborted;
1046 {
1047 SuspendibleThreadSetLeaver sts_leave(concurrent());
1048 barrier_aborted = !_first_overflow_barrier_sync.enter();
1049 }
1050
1051 // at this point everyone should have synced up and not be doing any
1052 // more work
1053
1054 if (barrier_aborted) {
1055 // If the barrier aborted we ignore the overflow condition and
1056 // just abort the whole marking phase as quickly as possible.
1057 return;
1058 }
1059 }
1060
1061 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
1062 SuspendibleThreadSetLeaver sts_leave(concurrent());
1063 _second_overflow_barrier_sync.enter();
1064
1065 // at this point everything should be re-initialized and ready to go
1066 }
1067
1068 class G1CMConcurrentMarkingTask : public WorkerTask {
1069 G1ConcurrentMark* _cm;
1070
1071 public:
1072 void work(uint worker_id) {
1073 ResourceMark rm;
1074
1075 SuspendibleThreadSetJoiner sts_join;
1076
1077 assert(worker_id < _cm->active_tasks(), "invariant");
1078
1079 G1CMTask* task = _cm->task(worker_id);
1080 task->record_start_time();
1081 if (!_cm->has_aborted()) {
1082 do {
1083 task->do_marking_step(G1ConcMarkStepDurationMillis,
1084 true /* do_termination */,
1085 false /* is_serial*/);
1086
1087 _cm->do_yield_check();
1088 } while (!_cm->has_aborted() && task->has_aborted());
1089 }
1090 task->record_end_time();
1091 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant");
1092 }
1093
1094 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm) :
1095 WorkerTask("Concurrent Mark"), _cm(cm) { }
1096
1097 ~G1CMConcurrentMarkingTask() { }
1098 };
1099
1100 uint G1ConcurrentMark::calc_active_marking_workers() {
1101 uint result = 0;
1102 if (!UseDynamicNumberOfGCThreads || !FLAG_IS_DEFAULT(ConcGCThreads)) {
1103 result = _max_concurrent_workers;
1104 } else {
1105 result =
1106 WorkerPolicy::calc_default_active_workers(_max_concurrent_workers,
1107 1, /* Minimum workers */
1108 _num_concurrent_workers,
1109 Threads::number_of_non_daemon_threads());
1110 // Don't scale the result down by scale_concurrent_workers() because
1111 // that scaling has already gone into "_max_concurrent_workers".
1112 }
1113 assert(result > 0 && result <= _max_concurrent_workers,
1114 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u",
1115 _max_concurrent_workers, result);
1116 return result;
1117 }
1118
1119 bool G1ConcurrentMark::has_root_region_scan_aborted() const {
1120 return _root_region_scan_aborted.load_relaxed();
1121 }
1122
1123 #ifndef PRODUCT
1124 void G1ConcurrentMark::assert_root_region_scan_completed_or_aborted() {
1125 assert(root_regions()->work_completed() || has_root_region_scan_aborted(), "must be");
1126 }
1127 #endif
1128
1129 void G1ConcurrentMark::scan_root_region(const MemRegion* region, uint worker_id) {
1130 #ifdef ASSERT
1131 HeapWord* last = region->last();
1132 G1HeapRegion* hr = _g1h->heap_region_containing(last);
1133 assert(hr->is_old() || top_at_mark_start(hr) == hr->bottom(),
1134 "Root regions must be old or survivor/eden but region %u is %s", hr->hrm_index(), hr->get_type_str());
1135 assert(top_at_mark_start(hr) == region->start(),
1136 "MemRegion start should be equal to TAMS");
1137 #endif
1138
1139 G1RootRegionScanClosure cl(_g1h, this, worker_id);
1140
1141 const uintx interval = PrefetchScanIntervalInBytes;
1142 HeapWord* curr = region->start();
1143 const HeapWord* end = region->end();
1144 while (curr < end) {
1145 Prefetch::read(curr, interval);
1146 oop obj = cast_to_oop(curr);
1147 size_t size = obj->oop_iterate_size(&cl);
1148 assert(size == obj->size(), "sanity");
1149 curr += size;
1150 }
1151 }
1152
1153 class G1CMRootRegionScanTask : public WorkerTask {
1154 G1ConcurrentMark* _cm;
1155 bool _should_yield;
1156
1157 public:
1158 G1CMRootRegionScanTask(G1ConcurrentMark* cm, bool should_yield) :
1159 WorkerTask("G1 Root Region Scan"), _cm(cm), _should_yield(should_yield) { }
1160
1161 void work(uint worker_id) {
1162 SuspendibleThreadSetJoiner sts_join(_should_yield);
1163
1164 while (true) {
1165 if (_cm->has_root_region_scan_aborted()) {
1166 return;
1167 }
1168 G1CMRootMemRegions* root_regions = _cm->root_regions();
1169 const MemRegion* region = root_regions->claim_next();
1170 if (region == nullptr) {
1171 return;
1172 }
1173 _cm->scan_root_region(region, worker_id);
1174 if (_should_yield) {
1175 SuspendibleThreadSet::yield();
1176 // If we yielded, a GC may have processed all root regions,
1177 // so this loop will naturally exit on the next claim_next() call.
1178 // Same if a Full GC signalled abort of the concurrent mark.
1179 }
1180 }
1181 }
1182 };
1183
1184 bool G1ConcurrentMark::scan_root_regions(WorkerThreads* workers, bool concurrent) {
1185 // We first check whether there is any work to do as we might have already aborted
1186 // the concurrent cycle, or ran into a GC that did the actual work when we reach here.
1187 // We want to avoid spinning up the worker threads if that happened.
1188 // (Note that due to races reading the abort-flag, we might spin up the threads anyway).
1189 //
1190 // Abort happens if a Full GC occurs right after starting the concurrent cycle or
1191 // a young gc doing the work.
1192 //
1193 // Concurrent gc threads enter an STS when starting the task, so they stop, then
1194 // continue after that safepoint.
1195 //
1196 // Must not use G1CMRootMemRegions::work_completed() here because we need to get a
1197 // consistent view of the value containing the number of remaining regions across the
1198 // usages below. The safepoint/gc may already be running and modifying it
1199 // while this code is still executing.
1200 uint num_remaining = root_regions()->num_remaining_regions();
1201 bool do_scan = num_remaining > 0 && !has_root_region_scan_aborted();
1202 if (do_scan) {
1203 // Assign one worker to each root-region but subject to the max constraint.
1204 // The constraint is also important to avoid accesses beyond the allocated per-worker
1205 // marking helper data structures. We might get passed different WorkerThreads with
1206 // different number of threads (potential worker ids) than helper data structures when
1207 // completing this work during GC.
1208 const uint num_workers = MIN2(num_remaining,
1209 _max_concurrent_workers);
1210
1211 G1CMRootRegionScanTask task(this, concurrent);
1212 log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
1213 task.name(), num_workers, num_remaining);
1214 workers->run_task(&task, num_workers);
1215 }
1216
1217 // At the end of this method, we can re-read num_remaining() in the assert: either
1218 // we got non-zero above and we processed all root regions (and it must be zero
1219 // after the worker task synchronization) or it had already been zero. We also
1220 // can't have started another concurrent cycle that could have set it to something else
1221 // while still in the concurrent cycle (if called concurrently).
1222 assert_root_region_scan_completed_or_aborted();
1223
1224 return do_scan;
1225 }
1226
1227 void G1ConcurrentMark::scan_root_regions_concurrently() {
1228 assert(Thread::current() == cm_thread(), "must be on Concurrent Mark Thread");
1229 scan_root_regions(_concurrent_workers, true /* concurrent */);
1230 }
1231
1232 bool G1ConcurrentMark::complete_root_regions_scan_in_safepoint() {
1233 assert_at_safepoint_on_vm_thread();
1234 return scan_root_regions(_g1h->workers(), false /* concurrent */);
1235 }
1236
1237 void G1ConcurrentMark::add_root_region(G1HeapRegion* r) {
1238 root_regions()->add(top_at_mark_start(r), r->top());
1239 }
1240
1241 void G1ConcurrentMark::add_root_region_set_bottom(G1HeapRegion* r) {
1242 set_top_at_mark_start_to_bottom(r);
1243 root_regions()->add(r->bottom(), r->top());
1244 }
1245
1246 bool G1ConcurrentMark::is_root_region(G1HeapRegion* r) {
1247 return root_regions()->contains(MemRegion(top_at_mark_start(r), r->top()));
1248 }
1249
1250 void G1ConcurrentMark::abort_root_region_scan() {
1251 assert_not_at_safepoint();
1252
1253 _root_region_scan_aborted.store_relaxed(true);
1254 }
1255
1256 void G1ConcurrentMark::abort_root_region_scan_at_safepoint() {
1257 assert_at_safepoint_on_vm_thread();
1258
1259 _root_region_scan_aborted.store_relaxed(true);
1260 }
1261
1262 void G1ConcurrentMark::concurrent_cycle_start() {
1263 _gc_timer_cm->register_gc_start();
1264
1265 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start());
1266
1267 _g1h->trace_heap_before_gc(_gc_tracer_cm);
1268 }
1269
1270 uint G1ConcurrentMark::completed_mark_cycles() const {
1271 return _completed_mark_cycles.load_relaxed();
1272 }
1273
1274 void G1ConcurrentMark::concurrent_cycle_end(bool mark_cycle_completed) {
1275 _g1h->trace_heap_after_gc(_gc_tracer_cm);
1276
1277 if (mark_cycle_completed) {
1278 _completed_mark_cycles.add_then_fetch(1u, memory_order_relaxed);
1279 }
1280
1281 if (has_aborted()) {
1282 log_info(gc, marking)("Concurrent Mark Abort");
1283 _gc_tracer_cm->report_concurrent_mode_failure();
1284 }
1285
1286 _gc_timer_cm->register_gc_end();
1287
1288 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1289 }
1290
1291 void G1ConcurrentMark::mark_from_roots() {
1292 _restart_for_overflow.store_relaxed(false);
1293
1294 uint active_workers = calc_active_marking_workers();
1295
1296 // Setting active workers is not guaranteed since fewer
1297 // worker threads may currently exist and more may not be
1298 // available.
1299 active_workers = _concurrent_workers->set_active_workers(active_workers);
1300 log_info(gc, task)("Concurrent Mark Using %u of %u Workers", active_workers, _concurrent_workers->max_workers());
1301
1302 _num_concurrent_workers = active_workers;
1303
1304 // Parallel task terminator is set in "set_concurrency_and_phase()"
1305 set_concurrency_and_phase(active_workers, true /* concurrent */);
1306
1307 G1CMConcurrentMarkingTask marking_task(this);
1308 _concurrent_workers->run_task(&marking_task);
1309 print_stats();
1310 }
1311
1312 const char* G1ConcurrentMark::verify_location_string(VerifyLocation location) {
1313 static const char* location_strings[] = { "Remark Before",
1314 "Remark After",
1315 "Remark Overflow",
1316 "Cleanup Before",
1317 "Cleanup After" };
1318 return location_strings[static_cast<std::underlying_type_t<VerifyLocation>>(location)];
1319 }
1320
1321 void G1ConcurrentMark::verify_during_pause(G1HeapVerifier::G1VerifyType type,
1322 VerifyLocation location) {
1323 G1HeapVerifier* verifier = _g1h->verifier();
1324
1325 verifier->verify_region_sets_optional();
1326
1327 const char* caller = verify_location_string(location);
1328
1329 if (VerifyDuringGC && G1HeapVerifier::should_verify(type)) {
1330 GCTraceTime(Debug, gc, phases) debug(caller, _gc_timer_cm);
1331
1332 size_t const BufLen = 512;
1333 char buffer[BufLen];
1334
1335 jio_snprintf(buffer, BufLen, "During GC (%s)", caller);
1336 verifier->verify(VerifyOption::G1UseConcMarking, buffer);
1337
1338 // Only check bitmap in Remark, and not at After-Verification because the regions
1339 // already have their TAMS'es reset.
1340 if (location != VerifyLocation::RemarkAfter) {
1341 verifier->verify_bitmap_clear(true /* above_tams_only */);
1342 }
1343 }
1344 }
1345
1346 class G1ObjectCountIsAliveClosure: public BoolObjectClosure {
1347 G1CollectedHeap* _g1h;
1348 public:
1349 G1ObjectCountIsAliveClosure(G1CollectedHeap* g1h) : _g1h(g1h) {}
1350
1351 bool do_object_b(oop obj) {
1352 return !_g1h->is_obj_dead(obj);
1353 }
1354 };
1355
1356 void G1ConcurrentMark::remark() {
1357 assert_at_safepoint_on_vm_thread();
1358
1359 // If a full collection has happened, we should not continue. However we might
1360 // have ended up here as the Remark VM operation has been scheduled already.
1361 if (has_aborted()) {
1362 return;
1363 }
1364
1365 G1Policy* policy = _g1h->policy();
1366 policy->record_pause_start_time();
1367
1368 double start = os::elapsedTime();
1369
1370 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyLocation::RemarkBefore);
1371
1372 {
1373 GCTraceTime(Debug, gc, phases) debug("Finalize Marking", _gc_timer_cm);
1374 finalize_marking();
1375 }
1376
1377 double mark_work_end = os::elapsedTime();
1378
1379 bool const mark_finished = !has_overflown();
1380 if (mark_finished) {
1381 weak_refs_work();
1382
1383 CodeCache::on_gc_marking_cycle_finish();
1384
1385 // Unload Klasses, String, Code Cache, etc.
1386 if (ClassUnloadingWithConcurrentMark) {
1387 G1CMIsAliveClosure is_alive(this);
1388 _g1h->unload_classes_and_code("Class Unloading", &is_alive, _gc_timer_cm);
1389 }
1390
1391 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
1392 // We're done with marking.
1393 // This is the end of the marking cycle, we're expected all
1394 // threads to have SATB queues with active set to true.
1395 satb_mq_set.set_active_all_threads(false, /* new active value */
1396 true /* expected_active */);
1397
1398 {
1399 GCTraceTime(Debug, gc, phases) debug("Flush Task Caches", _gc_timer_cm);
1400 flush_all_task_caches();
1401 }
1402
1403 // All marking completed. Check bitmap now as we will start to reset TAMSes
1404 // in parallel below so that we can not do this in the After-Remark verification.
1405 _g1h->verifier()->verify_bitmap_clear(true /* above_tams_only */);
1406
1407 {
1408 GCTraceTime(Debug, gc, phases) debug("Select For Rebuild and Reclaim Empty Regions", _gc_timer_cm);
1409
1410 G1UpdateRegionLivenessAndSelectForRebuildTask cl(_g1h, this, _g1h->workers()->active_workers());
1411 uint const num_workers = MIN2(G1UpdateRegionLivenessAndSelectForRebuildTask::desired_num_workers(_g1h->num_committed_regions()),
1412 _g1h->workers()->active_workers());
1413 log_debug(gc,ergo)("Running %s using %u workers for %u regions in heap", cl.name(), num_workers, _g1h->num_committed_regions());
1414 _g1h->workers()->run_task(&cl, num_workers);
1415
1416 log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u",
1417 _g1h->num_committed_regions(), cl.total_selected_for_rebuild());
1418
1419 _needs_remembered_set_rebuild = (cl.total_selected_for_rebuild() > 0);
1420
1421 if (_needs_remembered_set_rebuild) {
1422 GrowableArrayCHeap<G1HeapRegion*, mtGC>* selected = cl.sort_and_prune_old_selected();
1423 _g1h->policy()->candidates()->set_candidates_from_marking(selected);
1424 }
1425 }
1426
1427 if (log_is_enabled(Trace, gc, liveness)) {
1428 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1429 _g1h->heap_region_iterate(&cl);
1430 }
1431
1432 // Potentially, some empty-regions have been reclaimed; make this a
1433 // "collection" so that pending allocation can retry before attempting a
1434 // GC pause.
1435 _g1h->increment_total_collections();
1436
1437 // For Remark Pauses that may have been triggered by PeriodicGCs, we maintain
1438 // resizing based on MinHeapFreeRatio or MaxHeapFreeRatio. If a PeriodicGC is
1439 // triggered, it likely means there are very few regular GCs, making resizing
1440 // based on gc heuristics less effective.
1441 if (_g1h->last_gc_was_periodic()) {
1442 _g1h->resize_heap_after_full_collection(0 /* allocation_word_size */);
1443 }
1444
1445 compute_new_sizes();
1446
1447 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyLocation::RemarkAfter);
1448
1449 assert(!restart_for_overflow(), "sanity");
1450 // Completely reset the marking state (except bitmaps) since marking completed.
1451 reset_at_marking_complete();
1452
1453 CodeCache::arm_all_nmethods();
1454
1455 {
1456 GCTraceTime(Debug, gc, phases) debug("Report Object Count", _gc_timer_cm);
1457 G1ObjectCountIsAliveClosure is_alive(_g1h);
1458 _gc_tracer_cm->report_object_count_after_gc(&is_alive, _g1h->workers());
1459 }
1460
1461 // Successfully completed marking, advance state.
1462 cm_thread()->set_full_cycle_rebuild_and_scrub();
1463 } else {
1464 // We overflowed. Restart concurrent marking.
1465 _restart_for_overflow.store_relaxed(true);
1466
1467 verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyLocation::RemarkOverflow);
1468
1469 // Clear the marking state because we will be restarting
1470 // marking due to overflowing the global mark stack.
1471 reset_marking_for_restart();
1472 }
1473
1474 // Statistics
1475 double now = os::elapsedTime();
1476 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1477 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1478 _remark_times.add((now - start) * 1000.0);
1479
1480 _g1h->update_perf_counter_cpu_time();
1481
1482 policy->record_concurrent_mark_remark_end();
1483
1484 return;
1485 }
1486
1487 void G1ConcurrentMark::compute_new_sizes() {
1488 MetaspaceGC::compute_new_size();
1489
1490 // Cleanup will have freed any regions completely full of garbage.
1491 // Update the soft reference policy with the new heap occupancy.
1492 Universe::heap()->update_capacity_and_used_at_gc();
1493
1494 // We reclaimed old regions so we should calculate the sizes to make
1495 // sure we update the old gen/space data.
1496 _g1h->monitoring_support()->update_sizes();
1497 }
1498
1499 class G1UpdateRegionsAfterRebuild : public G1HeapRegionClosure {
1500 G1CollectedHeap* _g1h;
1501
1502 public:
1503 G1UpdateRegionsAfterRebuild(G1CollectedHeap* g1h) : _g1h(g1h) { }
1504
1505 bool do_heap_region(G1HeapRegion* r) override {
1506 // Update the remset tracking state from updating to complete
1507 // if remembered sets have been rebuilt.
1508 _g1h->policy()->remset_tracker()->update_after_rebuild(r);
1509 return false;
1510 }
1511 };
1512
1513 void G1ConcurrentMark::cleanup() {
1514 assert_at_safepoint_on_vm_thread();
1515
1516 // If a full collection has happened, we shouldn't do this.
1517 if (has_aborted()) {
1518 return;
1519 }
1520
1521 G1Policy* policy = _g1h->policy();
1522 policy->record_pause_start_time();
1523
1524 double start = os::elapsedTime();
1525
1526 verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyLocation::CleanupBefore);
1527
1528 if (needs_remembered_set_rebuild()) {
1529 // Update the remset tracking information as well as marking all regions
1530 // as fully parsable.
1531 GCTraceTime(Debug, gc, phases) debug("Update Remembered Set Tracking After Rebuild", _gc_timer_cm);
1532 G1UpdateRegionsAfterRebuild cl(_g1h);
1533 _g1h->heap_region_iterate(&cl);
1534 } else {
1535 log_debug(gc, phases)("No Remembered Sets to update after rebuild");
1536 }
1537
1538 verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyLocation::CleanupAfter);
1539
1540 // Local statistics
1541 _cleanup_times.add((os::elapsedTime() - start) * 1000.0);
1542
1543 {
1544 GCTraceTime(Debug, gc, phases) debug("Finalize Concurrent Mark Cleanup", _gc_timer_cm);
1545 policy->record_concurrent_mark_cleanup_end(needs_remembered_set_rebuild());
1546 }
1547
1548 // Advance state.
1549 cm_thread()->set_full_cycle_reset_for_next_cycle();
1550 return;
1551 }
1552
1553 // 'Keep Alive' oop closure used by both serial parallel reference processing.
1554 // Uses the G1CMTask associated with a worker thread (for serial reference
1555 // processing the G1CMTask for worker 0 is used) to preserve (mark) and
1556 // trace referent objects.
1557 //
1558 // Using the G1CMTask and embedded local queues avoids having the worker
1559 // threads operating on the global mark stack. This reduces the risk
1560 // of overflowing the stack - which we would rather avoid at this late
1561 // state. Also using the tasks' local queues removes the potential
1562 // of the workers interfering with each other that could occur if
1563 // operating on the global stack.
1564
1565 class G1CMKeepAliveAndDrainClosure : public OopClosure {
1566 G1ConcurrentMark* _cm;
1567 G1CMTask* _task;
1568 uint _ref_counter_limit;
1569 uint _ref_counter;
1570 bool _is_serial;
1571 public:
1572 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1573 _cm(cm), _task(task), _ref_counter_limit(G1RefProcDrainInterval),
1574 _ref_counter(_ref_counter_limit), _is_serial(is_serial) {
1575 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1576 }
1577
1578 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1579 virtual void do_oop( oop* p) { do_oop_work(p); }
1580
1581 template <class T> void do_oop_work(T* p) {
1582 if (_cm->has_overflown()) {
1583 return;
1584 }
1585 if (!_task->deal_with_reference(p)) {
1586 // We did not add anything to the mark bitmap (or mark stack), so there is
1587 // no point trying to drain it.
1588 return;
1589 }
1590 _ref_counter--;
1591
1592 if (_ref_counter == 0) {
1593 // We have dealt with _ref_counter_limit references, pushing them
1594 // and objects reachable from them on to the local stack (and
1595 // possibly the global stack). Call G1CMTask::do_marking_step() to
1596 // process these entries.
1597 //
1598 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if
1599 // there's nothing more to do (i.e. we're done with the entries that
1600 // were pushed as a result of the G1CMTask::deal_with_reference() calls
1601 // above) or we overflow.
1602 //
1603 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1604 // flag while there may still be some work to do. (See the comment at
1605 // the beginning of G1CMTask::do_marking_step() for those conditions -
1606 // one of which is reaching the specified time target.) It is only
1607 // when G1CMTask::do_marking_step() returns without setting the
1608 // has_aborted() flag that the marking step has completed.
1609 do {
1610 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1611 _task->do_marking_step(mark_step_duration_ms,
1612 false /* do_termination */,
1613 _is_serial);
1614 } while (_task->has_aborted() && !_cm->has_overflown());
1615 _ref_counter = _ref_counter_limit;
1616 }
1617 }
1618 };
1619
1620 // 'Drain' oop closure used by both serial and parallel reference processing.
1621 // Uses the G1CMTask associated with a given worker thread (for serial
1622 // reference processing the G1CMtask for worker 0 is used). Calls the
1623 // do_marking_step routine, with an unbelievably large timeout value,
1624 // to drain the marking data structures of the remaining entries
1625 // added by the 'keep alive' oop closure above.
1626
1627 class G1CMDrainMarkingStackClosure : public VoidClosure {
1628 G1ConcurrentMark* _cm;
1629 G1CMTask* _task;
1630 bool _is_serial;
1631 public:
1632 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1633 _cm(cm), _task(task), _is_serial(is_serial) {
1634 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1635 }
1636
1637 void do_void() {
1638 do {
1639 // We call G1CMTask::do_marking_step() to completely drain the local
1640 // and global marking stacks of entries pushed by the 'keep alive'
1641 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above).
1642 //
1643 // G1CMTask::do_marking_step() is called in a loop, which we'll exit
1644 // if there's nothing more to do (i.e. we've completely drained the
1645 // entries that were pushed as a result of applying the 'keep alive'
1646 // closure to the entries on the discovered ref lists) or we overflow
1647 // the global marking stack.
1648 //
1649 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1650 // flag while there may still be some work to do. (See the comment at
1651 // the beginning of G1CMTask::do_marking_step() for those conditions -
1652 // one of which is reaching the specified time target.) It is only
1653 // when G1CMTask::do_marking_step() returns without setting the
1654 // has_aborted() flag that the marking step has completed.
1655
1656 _task->do_marking_step(1000000000.0 /* something very large */,
1657 true /* do_termination */,
1658 _is_serial);
1659 } while (_task->has_aborted() && !_cm->has_overflown());
1660 }
1661 };
1662
1663 class G1CMRefProcProxyTask : public RefProcProxyTask {
1664 G1CollectedHeap& _g1h;
1665 G1ConcurrentMark& _cm;
1666
1667 public:
1668 G1CMRefProcProxyTask(uint max_workers, G1CollectedHeap& g1h, G1ConcurrentMark &cm)
1669 : RefProcProxyTask("G1CMRefProcProxyTask", max_workers),
1670 _g1h(g1h),
1671 _cm(cm) {}
1672
1673 void work(uint worker_id) override {
1674 assert(worker_id < _max_workers, "sanity");
1675 G1CMIsAliveClosure is_alive(&_cm);
1676 uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id;
1677 G1CMKeepAliveAndDrainClosure keep_alive(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single);
1678 BarrierEnqueueDiscoveredFieldClosure enqueue;
1679 G1CMDrainMarkingStackClosure complete_gc(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single);
1680 _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc);
1681 }
1682
1683 void prepare_run_task_hook() override {
1684 // We need to reset the concurrency level before each
1685 // proxy task execution, so that the termination protocol
1686 // and overflow handling in G1CMTask::do_marking_step() knows
1687 // how many workers to wait for.
1688 _cm.set_concurrency(_queue_count);
1689 }
1690 };
1691
1692 void G1ConcurrentMark::weak_refs_work() {
1693 ResourceMark rm;
1694
1695 {
1696 GCTraceTime(Debug, gc, phases) debug("Reference Processing", _gc_timer_cm);
1697
1698 ReferenceProcessor* rp = _g1h->ref_processor_cm();
1699
1700 // See the comment in G1CollectedHeap::ref_processing_init()
1701 // about how reference processing currently works in G1.
1702
1703 assert(_global_mark_stack.is_empty(), "mark stack should be empty");
1704
1705 // Prefer to grow the stack until the max capacity.
1706 _global_mark_stack.set_should_grow();
1707
1708 // Parallel processing task executor.
1709 G1CMRefProcProxyTask task(rp->max_num_queues(), *_g1h, *this);
1710 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->max_num_queues());
1711
1712 // Process the weak references.
1713 const ReferenceProcessorStats& stats = rp->process_discovered_references(task, _g1h->workers(), pt);
1714 _gc_tracer_cm->report_gc_reference_stats(stats);
1715 pt.print_all_references();
1716
1717 // The do_oop work routines of the keep_alive and drain_marking_stack
1718 // oop closures will set the has_overflown flag if we overflow the
1719 // global marking stack.
1720
1721 assert(has_overflown() || _global_mark_stack.is_empty(),
1722 "Mark stack should be empty (unless it has overflown)");
1723 }
1724
1725 if (has_overflown()) {
1726 // We can not trust g1_is_alive and the contents of the heap if the marking stack
1727 // overflowed while processing references. Exit the VM.
1728 fatal("Overflow during reference processing, can not continue. Current mark stack depth: "
1729 "%zu, MarkStackSize: %zu, MarkStackSizeMax: %zu. "
1730 "Please increase MarkStackSize and/or MarkStackSizeMax and restart.",
1731 _global_mark_stack.size(), MarkStackSize, MarkStackSizeMax);
1732 return;
1733 }
1734
1735 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1736
1737 {
1738 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm);
1739 G1CMIsAliveClosure is_alive(this);
1740 WeakProcessor::weak_oops_do(_g1h->workers(), &is_alive, &do_nothing_cl, 1);
1741 }
1742 }
1743
1744 class G1PrecleanYieldClosure : public YieldClosure {
1745 G1ConcurrentMark* _cm;
1746
1747 public:
1748 G1PrecleanYieldClosure(G1ConcurrentMark* cm) : _cm(cm) { }
1749
1750 virtual bool should_return() {
1751 return _cm->has_aborted();
1752 }
1753
1754 virtual bool should_return_fine_grain() {
1755 _cm->do_yield_check();
1756 return _cm->has_aborted();
1757 }
1758 };
1759
1760 void G1ConcurrentMark::preclean() {
1761 assert(G1UseReferencePrecleaning, "Precleaning must be enabled.");
1762
1763 SuspendibleThreadSetJoiner joiner;
1764
1765 BarrierEnqueueDiscoveredFieldClosure enqueue;
1766
1767 set_concurrency_and_phase(1, true);
1768
1769 G1PrecleanYieldClosure yield_cl(this);
1770
1771 ReferenceProcessor* rp = _g1h->ref_processor_cm();
1772 // Precleaning is single threaded. Temporarily disable MT discovery.
1773 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false);
1774 rp->preclean_discovered_references(rp->is_alive_non_header(),
1775 &enqueue,
1776 &yield_cl,
1777 _gc_timer_cm);
1778 }
1779
1780 // Closure for marking entries in SATB buffers.
1781 class G1CMSATBBufferClosure : public SATBBufferClosure {
1782 private:
1783 G1CMTask* _task;
1784 G1CollectedHeap* _g1h;
1785
1786 // This is very similar to G1CMTask::deal_with_reference, but with
1787 // more relaxed requirements for the argument, so this must be more
1788 // circumspect about treating the argument as an object.
1789 void do_entry(void* entry) const {
1790 _task->increment_refs_reached();
1791 oop const obj = cast_to_oop(entry);
1792 _task->make_reference_grey(obj);
1793 }
1794
1795 public:
1796 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
1797 : _task(task), _g1h(g1h) { }
1798
1799 virtual void do_buffer(void** buffer, size_t size) {
1800 for (size_t i = 0; i < size; ++i) {
1801 do_entry(buffer[i]);
1802 }
1803 }
1804 };
1805
1806 class G1RemarkThreadsClosure : public ThreadClosure {
1807 G1SATBMarkQueueSet& _qset;
1808
1809 public:
1810 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) :
1811 _qset(G1BarrierSet::satb_mark_queue_set()) {}
1812
1813 void do_thread(Thread* thread) {
1814 // Transfer any partial buffer to the qset for completed buffer processing.
1815 _qset.flush_queue(G1ThreadLocalData::satb_mark_queue(thread));
1816 }
1817 };
1818
1819 class G1CMRemarkTask : public WorkerTask {
1820 // For Threads::possibly_parallel_threads_do
1821 ThreadsClaimTokenScope _threads_claim_token_scope;
1822 G1ConcurrentMark* _cm;
1823 public:
1824 void work(uint worker_id) {
1825 G1CMTask* task = _cm->task(worker_id);
1826 task->record_start_time();
1827 {
1828 ResourceMark rm;
1829
1830 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
1831 Threads::possibly_parallel_threads_do(true /* is_par */, &threads_f);
1832 }
1833
1834 do {
1835 task->do_marking_step(1000000000.0 /* something very large */,
1836 true /* do_termination */,
1837 false /* is_serial */);
1838 } while (task->has_aborted() && !_cm->has_overflown());
1839 // If we overflow, then we do not want to restart. We instead
1840 // want to abort remark and do concurrent marking again.
1841 task->record_end_time();
1842 }
1843
1844 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) :
1845 WorkerTask("Par Remark"), _threads_claim_token_scope(), _cm(cm) {
1846 _cm->terminator()->reset_for_reuse(active_workers);
1847 }
1848 };
1849
1850 void G1ConcurrentMark::finalize_marking() {
1851 ResourceMark rm;
1852
1853 _g1h->ensure_parsability(false);
1854
1855 // this is remark, so we'll use up all active threads
1856 uint active_workers = _g1h->workers()->active_workers();
1857 set_concurrency_and_phase(active_workers, false /* concurrent */);
1858 // Leave _parallel_marking_threads at it's
1859 // value originally calculated in the G1ConcurrentMark
1860 // constructor and pass values of the active workers
1861 // through the task.
1862
1863 {
1864 G1CMRemarkTask remarkTask(this, active_workers);
1865 // We will start all available threads, even if we decide that the
1866 // active_workers will be fewer. The extra ones will just bail out
1867 // immediately.
1868 _g1h->workers()->run_task(&remarkTask);
1869 }
1870
1871 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
1872 guarantee(has_overflown() ||
1873 satb_mq_set.completed_buffers_num() == 0,
1874 "Invariant: has_overflown = %s, num buffers = %zu",
1875 BOOL_TO_STR(has_overflown()),
1876 satb_mq_set.completed_buffers_num());
1877
1878 print_stats();
1879 }
1880
1881 void G1ConcurrentMark::flush_all_task_caches(bool ends_use_of_mark_cache) {
1882 size_t hits = 0;
1883 size_t misses = 0;
1884 for (uint i = 0; i < _max_num_tasks; i++) {
1885 Pair<size_t, size_t> stats = _tasks[i]->flush_mark_stats_cache();
1886 hits += stats.first;
1887 misses += stats.second;
1888 }
1889 size_t sum = hits + misses;
1890 log_debug(gc, stats)("Mark stats cache hits %zu misses %zu ratio %1.3lf",
1891 hits, misses, percent_of(hits, sum));
1892 if (ends_use_of_mark_cache) {
1893 _is_region_mark_stats_cache_in_use = false;
1894 }
1895 }
1896
1897 void G1ConcurrentMark::clear_bitmap_for_region(G1HeapRegion* hr) {
1898 assert_at_safepoint();
1899 _mark_bitmap.clear_range(MemRegion(hr->bottom(), hr->end()));
1900 }
1901
1902 G1HeapRegion* G1ConcurrentMark::claim_region(uint worker_id) {
1903 // "Checkpoint" the finger.
1904 HeapWord* local_finger = finger();
1905
1906 while (local_finger < _heap.end()) {
1907 assert(_g1h->is_in_reserved(local_finger), "invariant");
1908
1909 G1HeapRegion* curr_region = _g1h->heap_region_containing_or_null(local_finger);
1910 // Make sure that the reads below do not float before loading curr_region.
1911 OrderAccess::loadload();
1912 // Above heap_region_containing may return null as we always scan claim
1913 // until the end of the heap. In this case, just jump to the next region.
1914 HeapWord* end = curr_region != nullptr ? curr_region->end() : local_finger + G1HeapRegion::GrainWords;
1915
1916 // Is the gap between reading the finger and doing the CAS too long?
1917 HeapWord* res = _finger.compare_exchange(local_finger, end);
1918 if (res == local_finger && curr_region != nullptr) {
1919 // We succeeded.
1920 HeapWord* bottom = curr_region->bottom();
1921 HeapWord* limit = top_at_mark_start(curr_region);
1922
1923 log_trace(gc, marking)("Claim region %u bottom " PTR_FORMAT " tams " PTR_FORMAT, curr_region->hrm_index(), p2i(curr_region->bottom()), p2i(top_at_mark_start(curr_region)));
1924 // Notice that _finger == end cannot be guaranteed here since,
1925 // someone else might have moved the finger even further.
1926 assert(finger() >= end, "The finger should have moved forward");
1927
1928 if (limit > bottom) {
1929 return curr_region;
1930 } else {
1931 assert(limit == bottom,
1932 "The scan limit for region %u (%s) should be bottom but is " PTR_FORMAT,
1933 curr_region->hrm_index(), curr_region->get_short_type_str(), p2i(limit));
1934 // We return null and the caller should try calling
1935 // claim_region() again.
1936 return nullptr;
1937 }
1938 } else {
1939 // Read the finger again.
1940 HeapWord* next_finger = finger();
1941 assert(next_finger > local_finger,
1942 "The finger should have moved forward " PTR_FORMAT " " PTR_FORMAT,
1943 p2i(local_finger), p2i(next_finger));
1944 local_finger = next_finger;
1945 }
1946 }
1947
1948 return nullptr;
1949 }
1950
1951 #ifndef PRODUCT
1952 class VerifyNoCSetOops {
1953 G1CollectedHeap* _g1h;
1954 const char* _phase;
1955 int _info;
1956
1957 public:
1958 VerifyNoCSetOops(const char* phase, int info = -1) :
1959 _g1h(G1CollectedHeap::heap()),
1960 _phase(phase),
1961 _info(info)
1962 { }
1963
1964 void operator()(G1TaskQueueEntry task_entry) const {
1965 if (task_entry.is_partial_array_state()) {
1966 oop obj = task_entry.to_partial_array_state()->source();
1967 guarantee(_g1h->is_in_reserved(obj), "Partial Array " PTR_FORMAT " must be in heap.", p2i(obj));
1968 return;
1969 }
1970 guarantee(oopDesc::is_oop(task_entry.to_oop()),
1971 "Non-oop " PTR_FORMAT ", phase: %s, info: %d",
1972 p2i(task_entry.to_oop()), _phase, _info);
1973 G1HeapRegion* r = _g1h->heap_region_containing(task_entry.to_oop());
1974 guarantee(!(r->in_collection_set() || r->has_index_in_opt_cset()),
1975 "obj " PTR_FORMAT " from %s (%d) in region %u in (optional) collection set",
1976 p2i(task_entry.to_oop()), _phase, _info, r->hrm_index());
1977 }
1978 };
1979
1980 void G1ConcurrentMark::verify_no_collection_set_oops() {
1981 assert(SafepointSynchronize::is_at_safepoint() || !is_init_completed(),
1982 "should be at a safepoint or initializing");
1983 if (!is_fully_initialized() || !_g1h->collector_state()->is_in_mark_or_rebuild()) {
1984 return;
1985 }
1986
1987 // Verify entries on the global mark stack
1988 _global_mark_stack.iterate(VerifyNoCSetOops("Stack"));
1989
1990 // Verify entries on the task queues
1991 for (uint i = 0; i < _max_num_tasks; ++i) {
1992 G1CMTaskQueue* queue = _task_queues->queue(i);
1993 queue->iterate(VerifyNoCSetOops("Queue", i));
1994 }
1995
1996 // Verify the global finger
1997 HeapWord* global_finger = finger();
1998 if (global_finger != nullptr && global_finger < _heap.end()) {
1999 // Since we always iterate over all regions, we might get a null G1HeapRegion
2000 // here.
2001 G1HeapRegion* global_hr = _g1h->heap_region_containing_or_null(global_finger);
2002 guarantee(global_hr == nullptr || global_finger == global_hr->bottom(),
2003 "global finger: " PTR_FORMAT " region: " HR_FORMAT,
2004 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
2005 }
2006
2007 // Verify the task fingers
2008 assert(_num_concurrent_workers <= _max_num_tasks, "sanity");
2009 for (uint i = 0; i < _num_concurrent_workers; ++i) {
2010 G1CMTask* task = _tasks[i];
2011 HeapWord* task_finger = task->finger();
2012 if (task_finger != nullptr && task_finger < _heap.end()) {
2013 // See above note on the global finger verification.
2014 G1HeapRegion* r = _g1h->heap_region_containing_or_null(task_finger);
2015 guarantee(r == nullptr || task_finger == r->bottom() ||
2016 !r->in_collection_set() || !r->has_index_in_opt_cset(),
2017 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
2018 p2i(task_finger), HR_FORMAT_PARAMS(r));
2019 }
2020 }
2021 }
2022 #endif // PRODUCT
2023
2024 void G1ConcurrentMark::rebuild_and_scrub() {
2025 if (!needs_remembered_set_rebuild()) {
2026 log_debug(gc, marking)("Skipping Remembered Set Rebuild. No regions selected for rebuild, will only scrub");
2027 }
2028
2029 G1ConcurrentRebuildAndScrub::rebuild_and_scrub(this, needs_remembered_set_rebuild(), _concurrent_workers);
2030 }
2031
2032 void G1ConcurrentMark::print_stats() {
2033 if (!log_is_enabled(Debug, gc, stats)) {
2034 return;
2035 }
2036 log_debug(gc, stats)("---------------------------------------------------------------------");
2037 for (size_t i = 0; i < _num_active_tasks; ++i) {
2038 _tasks[i]->print_stats();
2039 log_debug(gc, stats)("---------------------------------------------------------------------");
2040 }
2041 }
2042
2043 bool G1ConcurrentMark::shutdown_cleanup_needed() const {
2044 // Cleanup (aborting threads, setting abort flags) is needed throughout the whole cycle before
2045 // stopping the CM thread.
2046 return is_fully_initialized() && is_in_concurrent_cycle();
2047 }
2048
2049 void G1ConcurrentMark::shutdown_concurrent_cycle() {
2050 assert_at_safepoint_on_vm_thread();
2051
2052 abort_root_region_scan_at_safepoint();
2053 abort_marking_threads();
2054
2055 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
2056 satb_mq_set.abandon_partial_marking();
2057 // This can be called either during or outside marking, we'll read
2058 // the expected_active value from the SATB queue set.
2059 satb_mq_set.set_active_all_threads(false, /* new active value */
2060 satb_mq_set.is_active() /* expected_active */);
2061 }
2062
2063 bool G1ConcurrentMark::concurrent_cycle_abort() {
2064 assert_at_safepoint_on_vm_thread();
2065 assert(_g1h->collector_state()->is_in_full_gc(), "must be");
2066
2067 // If we start the compaction before the CM threads finish
2068 // scanning the root regions we might trip them over as we'll
2069 // be moving objects / updating references. Since the root region
2070 // scan synchronized with the safepoint, just tell it to abort.
2071 // It will notice when the threads start up again later.
2072 abort_root_region_scan_at_safepoint();
2073
2074 // We haven't started a concurrent cycle no need to do anything; we might have
2075 // aborted the marking because of shutting down though. In this case the marking
2076 // might have already completed the abort (leading to in_progress() below to
2077 // return false), however this still left marking state particularly in the
2078 // shared marking bitmap that must be cleaned up.
2079 // If there are multiple full gcs during shutdown we do this work repeatedly for
2080 // nothing, but this situation should be extremely rare (a full gc after shutdown
2081 // has been signalled is already rare), and this work should be negligible compared
2082 // to actual full gc work.
2083
2084 if (!is_fully_initialized() || (!cm_thread()->is_in_progress() && !cm_thread()->should_terminate())) {
2085 return false;
2086 }
2087
2088 flush_all_task_caches();
2089 reset_marking_for_restart();
2090
2091 abort_marking_threads();
2092
2093 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
2094 satb_mq_set.abandon_partial_marking();
2095 // This can be called either during or outside marking, we'll read
2096 // the expected_active value from the SATB queue set.
2097 satb_mq_set.set_active_all_threads(false, /* new active value */
2098 satb_mq_set.is_active() /* expected_active */);
2099 return true;
2100 }
2101
2102 void G1ConcurrentMark::abort_marking_threads() {
2103 assert_root_region_scan_completed_or_aborted();
2104 _has_aborted.store_relaxed(true);
2105 _first_overflow_barrier_sync.abort();
2106 _second_overflow_barrier_sync.abort();
2107 }
2108
2109 double G1ConcurrentMark::worker_threads_cpu_time_s() {
2110 class CountCpuTimeThreadClosure : public ThreadClosure {
2111 public:
2112 jlong _total_cpu_time;
2113
2114 CountCpuTimeThreadClosure() : ThreadClosure(), _total_cpu_time(0) { }
2115
2116 void do_thread(Thread* t) {
2117 _total_cpu_time += os::thread_cpu_time(t);
2118 }
2119 } cl;
2120
2121 threads_do(&cl);
2122
2123 return (double)cl._total_cpu_time / NANOSECS_PER_SEC;
2124 }
2125
2126 static void print_ms_time_info(const char* prefix, const char* name,
2127 NumberSeq& ns) {
2128 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2129 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2130 if (ns.num() > 0) {
2131 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2132 prefix, ns.sd(), ns.maximum());
2133 }
2134 }
2135
2136 void G1ConcurrentMark::print_summary_info() {
2137 Log(gc, marking) log;
2138 if (!log.is_trace()) {
2139 return;
2140 }
2141
2142 log.trace(" Concurrent marking:");
2143 if (!is_fully_initialized()) {
2144 log.trace(" has not been initialized yet");
2145 return;
2146 }
2147 print_ms_time_info(" ", "remarks", _remark_times);
2148 {
2149 print_ms_time_info(" ", "final marks", _remark_mark_times);
2150 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2151
2152 }
2153 print_ms_time_info(" ", "cleanups", _cleanup_times);
2154 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
2155 _cleanup_times.sum() / 1000.0, _cleanup_times.avg());
2156 log.trace(" Total stop_world time = %8.2f s.",
2157 (_remark_times.sum() + _cleanup_times.sum())/1000.0);
2158 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2159 cm_thread()->total_mark_cpu_time_s(), cm_thread()->worker_threads_cpu_time_s());
2160 }
2161
2162 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2163 if (is_fully_initialized()) { // they are initialized late
2164 tc->do_thread(_cm_thread);
2165 _concurrent_workers->threads_do(tc);
2166 }
2167 }
2168
2169 void G1ConcurrentMark::print_on(outputStream* st) const {
2170 st->print_cr("Marking Bits: (CMBitMap*) " PTR_FORMAT, p2i(mark_bitmap()));
2171 _mark_bitmap.print_on(st, " Bits: ");
2172 }
2173
2174 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2175 ReferenceProcessor* result = g1h->ref_processor_cm();
2176 assert(result != nullptr, "CM reference processor should not be null");
2177 return result;
2178 }
2179
2180 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2181 G1CMTask* task)
2182 : ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_strong, get_cm_oop_closure_ref_processor(g1h)),
2183 _g1h(g1h), _task(task)
2184 { }
2185
2186 void G1CMTask::setup_for_region(G1HeapRegion* hr) {
2187 assert(hr != nullptr,
2188 "claim_region() should have filtered out null regions");
2189 _curr_region = hr;
2190 _finger = hr->bottom();
2191 update_region_limit();
2192 }
2193
2194 void G1CMTask::update_region_limit() {
2195 G1HeapRegion* hr = _curr_region;
2196 HeapWord* bottom = hr->bottom();
2197 HeapWord* limit = _cm->top_at_mark_start(hr);
2198
2199 if (limit == bottom) {
2200 // The region was collected underneath our feet.
2201 // We set the finger to bottom to ensure that the bitmap
2202 // iteration that will follow this will not do anything.
2203 // (this is not a condition that holds when we set the region up,
2204 // as the region is not supposed to be empty in the first place)
2205 _finger = bottom;
2206 } else if (limit >= _region_limit) {
2207 assert(limit >= _finger, "peace of mind");
2208 } else {
2209 assert(limit < _region_limit, "only way to get here");
2210 // This can happen under some pretty unusual circumstances. An
2211 // evacuation pause empties the region underneath our feet (TAMS
2212 // at bottom). We then do some allocation in the region (TAMS
2213 // stays at bottom), followed by the region being used as a GC
2214 // alloc region (TAMS will move to top() and the objects
2215 // originally below it will be greyed). All objects now marked in
2216 // the region are explicitly greyed, if below the global finger,
2217 // and we do not need in fact to scan anything else. So, we simply
2218 // set _finger to be limit to ensure that the bitmap iteration
2219 // doesn't do anything.
2220 _finger = limit;
2221 }
2222
2223 _region_limit = limit;
2224 }
2225
2226 void G1CMTask::giveup_current_region() {
2227 assert(_curr_region != nullptr, "invariant");
2228 clear_region_fields();
2229 }
2230
2231 void G1CMTask::clear_region_fields() {
2232 // Values for these three fields that indicate that we're not
2233 // holding on to a region.
2234 _curr_region = nullptr;
2235 _finger = nullptr;
2236 _region_limit = nullptr;
2237 }
2238
2239 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
2240 if (cm_oop_closure == nullptr) {
2241 assert(_cm_oop_closure != nullptr, "invariant");
2242 } else {
2243 assert(_cm_oop_closure == nullptr, "invariant");
2244 }
2245 _cm_oop_closure = cm_oop_closure;
2246 }
2247
2248 void G1CMTask::reset(G1CMBitMap* mark_bitmap) {
2249 guarantee(mark_bitmap != nullptr, "invariant");
2250 _mark_bitmap = mark_bitmap;
2251 clear_region_fields();
2252
2253 _calls = 0;
2254 _elapsed_time_ms = 0.0;
2255 _termination_time_ms = 0.0;
2256
2257 _mark_stats_cache.reset();
2258 }
2259
2260 void G1CMTask::reset_for_restart() {
2261 clear_region_fields();
2262 _task_queue->set_empty();
2263 TASKQUEUE_STATS_ONLY(_partial_array_splitter.stats()->reset());
2264 TASKQUEUE_STATS_ONLY(_task_queue->stats.reset());
2265 }
2266
2267 void G1CMTask::register_partial_array_splitter() {
2268
2269 ::new (&_partial_array_splitter) PartialArraySplitter(_cm->partial_array_state_manager(),
2270 _cm->max_num_tasks());
2271 }
2272
2273 void G1CMTask::unregister_partial_array_splitter() {
2274 _partial_array_splitter.~PartialArraySplitter();
2275 }
2276
2277 bool G1CMTask::should_exit_termination() {
2278 if (!regular_clock_call()) {
2279 return true;
2280 }
2281
2282 // This is called when we are in the termination protocol. We should
2283 // quit if, for some reason, this task wants to abort or the global
2284 // stack is not empty (this means that we can get work from it).
2285 return !_cm->mark_stack_empty() || has_aborted();
2286 }
2287
2288 void G1CMTask::reached_limit() {
2289 assert(_words_scanned >= _words_scanned_limit ||
2290 _refs_reached >= _refs_reached_limit ,
2291 "shouldn't have been called otherwise");
2292 abort_marking_if_regular_check_fail();
2293 }
2294
2295 bool G1CMTask::regular_clock_call() {
2296 if (has_aborted()) {
2297 return false;
2298 }
2299
2300 // First, we need to recalculate the words scanned and refs reached
2301 // limits for the next clock call.
2302 recalculate_limits();
2303
2304 // During the regular clock call we do the following
2305
2306 // (1) If an overflow has been flagged, then we abort.
2307 if (_cm->has_overflown()) {
2308 return false;
2309 }
2310
2311 // If we are not concurrent (i.e. we're doing remark) we don't need
2312 // to check anything else. The other steps are only needed during
2313 // the concurrent marking phase.
2314 if (!_cm->concurrent()) {
2315 return true;
2316 }
2317
2318 // (2) If marking has been aborted for Full GC, then we also abort.
2319 if (_cm->has_aborted()) {
2320 return false;
2321 }
2322
2323 // (4) We check whether we should yield. If we have to, then we abort.
2324 if (SuspendibleThreadSet::should_yield()) {
2325 // We should yield. To do this we abort the task. The caller is
2326 // responsible for yielding.
2327 return false;
2328 }
2329
2330 // (5) We check whether we've reached our time quota. If we have,
2331 // then we abort.
2332 double elapsed_time_ms = (double)(os::current_thread_cpu_time() - _start_cpu_time_ns) / NANOSECS_PER_MILLISEC;
2333 if (elapsed_time_ms > _time_target_ms) {
2334 _has_timed_out = true;
2335 return false;
2336 }
2337
2338 // (6) Finally, we check whether there are enough completed STAB
2339 // buffers available for processing. If there are, we abort.
2340 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
2341 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
2342 // we do need to process SATB buffers, we'll abort and restart
2343 // the marking task to do so
2344 return false;
2345 }
2346 return true;
2347 }
2348
2349 void G1CMTask::recalculate_limits() {
2350 _real_words_scanned_limit = _words_scanned + words_scanned_period;
2351 _words_scanned_limit = _real_words_scanned_limit;
2352
2353 _real_refs_reached_limit = _refs_reached + refs_reached_period;
2354 _refs_reached_limit = _real_refs_reached_limit;
2355 }
2356
2357 void G1CMTask::decrease_limits() {
2358 // This is called when we believe that we're going to do an infrequent
2359 // operation which will increase the per byte scanned cost (i.e. move
2360 // entries to/from the global stack). It basically tries to decrease the
2361 // scanning limit so that the clock is called earlier.
2362
2363 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4;
2364 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4;
2365 }
2366
2367 void G1CMTask::move_entries_to_global_stack() {
2368 // Local array where we'll store the entries that will be popped
2369 // from the local queue.
2370 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2371
2372 size_t n = 0;
2373 G1TaskQueueEntry task_entry;
2374 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) {
2375 buffer[n] = task_entry;
2376 ++n;
2377 }
2378 if (n < G1CMMarkStack::EntriesPerChunk) {
2379 buffer[n] = G1TaskQueueEntry();
2380 }
2381
2382 if (n > 0) {
2383 if (!_cm->mark_stack_push(buffer)) {
2384 set_has_aborted();
2385 }
2386 }
2387
2388 // This operation was quite expensive, so decrease the limits.
2389 decrease_limits();
2390 }
2391
2392 bool G1CMTask::get_entries_from_global_stack() {
2393 // Local array where we'll store the entries that will be popped
2394 // from the global stack.
2395 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2396
2397 if (!_cm->mark_stack_pop(buffer)) {
2398 return false;
2399 }
2400
2401 // We did actually pop at least one entry.
2402 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) {
2403 G1TaskQueueEntry task_entry = buffer[i];
2404 if (task_entry.is_null()) {
2405 break;
2406 }
2407 assert(task_entry.is_partial_array_state() || oopDesc::is_oop(task_entry.to_oop()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.to_oop()));
2408 bool success = _task_queue->push(task_entry);
2409 // We only call this when the local queue is empty or under a
2410 // given target limit. So, we do not expect this push to fail.
2411 assert(success, "invariant");
2412 }
2413
2414 // This operation was quite expensive, so decrease the limits
2415 decrease_limits();
2416 return true;
2417 }
2418
2419 void G1CMTask::drain_local_queue(bool partially) {
2420 if (has_aborted()) {
2421 return;
2422 }
2423
2424 // Decide what the target size is, depending whether we're going to
2425 // drain it partially (so that other tasks can steal if they run out
2426 // of things to do) or totally (at the very end).
2427 uint target_size;
2428 if (partially) {
2429 target_size = GCDrainStackTargetSize;
2430 } else {
2431 target_size = 0;
2432 }
2433
2434 if (_task_queue->size() > target_size) {
2435 G1TaskQueueEntry entry;
2436 bool ret = _task_queue->pop_local(entry);
2437 while (ret) {
2438 process_entry(entry, false /* stolen */);
2439 if (_task_queue->size() <= target_size || has_aborted()) {
2440 ret = false;
2441 } else {
2442 ret = _task_queue->pop_local(entry);
2443 }
2444 }
2445 }
2446 }
2447
2448 size_t G1CMTask::start_partial_array_processing(objArrayOop obj) {
2449 assert(obj->length() >= (int)ObjArrayMarkingStride, "Must be a large array object %d", obj->length());
2450
2451 // Mark klass metadata
2452 process_klass(obj->klass());
2453
2454 size_t array_length = obj->length();
2455 size_t initial_chunk_size = _partial_array_splitter.start(_task_queue, obj, nullptr, array_length, ObjArrayMarkingStride);
2456
2457 process_array_chunk(obj, 0, initial_chunk_size);
2458
2459 // Include object header size
2460 if (obj->is_refArray()) {
2461 return refArrayOopDesc::object_size(checked_cast<int>(initial_chunk_size));
2462 } else {
2463 FlatArrayKlass* fak = FlatArrayKlass::cast(obj->klass());
2464 return flatArrayOopDesc::object_size(fak->layout_helper(), checked_cast<int>(initial_chunk_size));
2465 }
2466 }
2467
2468 size_t G1CMTask::process_partial_array(const G1TaskQueueEntry& task, bool stolen) {
2469 PartialArrayState* state = task.to_partial_array_state();
2470 // Access state before release by claim().
2471 objArrayOop obj = oop_cast<objArrayOop>(state->source());
2472
2473 PartialArraySplitter::Claim claim =
2474 _partial_array_splitter.claim(state, _task_queue, stolen);
2475
2476 process_array_chunk(obj, claim._start, claim._end);
2477
2478 if (obj->is_refArray()) {
2479 return heap_word_size((claim._end - claim._start) * heapOopSize);
2480 } else {
2481 assert(obj->is_flatArray(), "Must be!");
2482 size_t element_byte_size = FlatArrayKlass::cast(obj->klass())->element_byte_size();
2483 size_t nof_elements = claim._end - claim._start;
2484 return heap_word_size(nof_elements * element_byte_size);
2485 }
2486 }
2487
2488 void G1CMTask::drain_global_stack(bool partially) {
2489 if (has_aborted()) {
2490 return;
2491 }
2492
2493 // We have a policy to drain the local queue before we attempt to
2494 // drain the global stack.
2495 assert(partially || _task_queue->size() == 0, "invariant");
2496
2497 // Decide what the target size is, depending whether we're going to
2498 // drain it partially (so that other tasks can steal if they run out
2499 // of things to do) or totally (at the very end).
2500 // Notice that when draining the global mark stack partially, due to the racyness
2501 // of the mark stack size update we might in fact drop below the target. But,
2502 // this is not a problem.
2503 // In case of total draining, we simply process until the global mark stack is
2504 // totally empty, disregarding the size counter.
2505 if (partially) {
2506 size_t const target_size = _cm->partial_mark_stack_size_target();
2507 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
2508 if (get_entries_from_global_stack()) {
2509 drain_local_queue(partially);
2510 }
2511 }
2512 } else {
2513 while (!has_aborted() && get_entries_from_global_stack()) {
2514 drain_local_queue(partially);
2515 }
2516 }
2517 }
2518
2519 // SATB Queue has several assumptions on whether to call the par or
2520 // non-par versions of the methods. this is why some of the code is
2521 // replicated. We should really get rid of the single-threaded version
2522 // of the code to simplify things.
2523 void G1CMTask::drain_satb_buffers() {
2524 if (has_aborted()) {
2525 return;
2526 }
2527
2528 // We set this so that the regular clock knows that we're in the
2529 // middle of draining buffers and doesn't set the abort flag when it
2530 // notices that SATB buffers are available for draining. It'd be
2531 // very counter productive if it did that. :-)
2532 _draining_satb_buffers = true;
2533
2534 G1CMSATBBufferClosure satb_cl(this, _g1h);
2535 SATBMarkQueueSet& satb_mq_set = G1BarrierSet::satb_mark_queue_set();
2536
2537 // This keeps claiming and applying the closure to completed buffers
2538 // until we run out of buffers or we need to abort.
2539 while (!has_aborted() &&
2540 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
2541 abort_marking_if_regular_check_fail();
2542 }
2543
2544 // Can't assert qset is empty here, even if not aborted. If concurrent,
2545 // some other thread might be adding to the queue. If not concurrent,
2546 // some other thread might have won the race for the last buffer, but
2547 // has not yet decremented the count.
2548
2549 _draining_satb_buffers = false;
2550
2551 // again, this was a potentially expensive operation, decrease the
2552 // limits to get the regular clock call early
2553 decrease_limits();
2554 }
2555
2556 #ifndef PRODUCT
2557 void G1CMTask::verify_no_mark_stats_for(uint region_idx) {
2558 _mark_stats_cache.verify_no_mark_stats_for(region_idx);
2559 }
2560 #endif
2561
2562 void G1CMTask::clear_mark_stats_cache(uint region_idx) {
2563 _mark_stats_cache.reset(region_idx);
2564 }
2565
2566 Pair<size_t, size_t> G1CMTask::flush_mark_stats_cache() {
2567 return _mark_stats_cache.evict_all();
2568 }
2569
2570 void G1CMTask::print_stats() {
2571 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u", _worker_id, _calls);
2572 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
2573 _elapsed_time_ms, _termination_time_ms);
2574 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms max = %1.2lfms, total = %1.2lfms",
2575 _step_times_ms.num(),
2576 _step_times_ms.avg(),
2577 _step_times_ms.sd(),
2578 _step_times_ms.maximum(),
2579 _step_times_ms.sum());
2580 size_t const hits = _mark_stats_cache.hits();
2581 size_t const misses = _mark_stats_cache.misses();
2582 log_debug(gc, stats)(" Mark Stats Cache: hits %zu misses %zu ratio %.3f",
2583 hits, misses, percent_of(hits, hits + misses));
2584 }
2585
2586 bool G1ConcurrentMark::try_stealing(uint worker_id, G1TaskQueueEntry& task_entry) {
2587 return _task_queues->steal(worker_id, task_entry);
2588 }
2589
2590 void G1CMTask::process_current_region(G1CMBitMapClosure& bitmap_closure) {
2591 if (has_aborted() || _curr_region == nullptr) {
2592 return;
2593 }
2594
2595 // This means that we're already holding on to a region.
2596 assert(_finger != nullptr, "if region is not null, then the finger "
2597 "should not be null either");
2598
2599 // We might have restarted this task after an evacuation pause
2600 // which might have evacuated the region we're holding on to
2601 // underneath our feet. Let's read its limit again to make sure
2602 // that we do not iterate over a region of the heap that
2603 // contains garbage (update_region_limit() will also move
2604 // _finger to the start of the region if it is found empty).
2605 update_region_limit();
2606 // We will start from _finger not from the start of the region,
2607 // as we might be restarting this task after aborting half-way
2608 // through scanning this region. In this case, _finger points to
2609 // the address where we last found a marked object. If this is a
2610 // fresh region, _finger points to start().
2611 MemRegion mr = MemRegion(_finger, _region_limit);
2612
2613 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2614 "humongous regions should go around loop once only");
2615
2616 // Some special cases:
2617 // If the memory region is empty, we can just give up the region.
2618 // If the current region is humongous then we only need to check
2619 // the bitmap for the bit associated with the start of the object,
2620 // scan the object if it's live, and give up the region.
2621 // Otherwise, let's iterate over the bitmap of the part of the region
2622 // that is left.
2623 // If the iteration is successful, give up the region.
2624 if (mr.is_empty()) {
2625 giveup_current_region();
2626 abort_marking_if_regular_check_fail();
2627 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2628 if (_mark_bitmap->is_marked(mr.start())) {
2629 // The object is marked - apply the closure
2630 bitmap_closure.do_addr(mr.start());
2631 }
2632 // Even if this task aborted while scanning the humongous object
2633 // we can (and should) give up the current region.
2634 giveup_current_region();
2635 abort_marking_if_regular_check_fail();
2636 } else if (_mark_bitmap->iterate(&bitmap_closure, mr)) {
2637 giveup_current_region();
2638 abort_marking_if_regular_check_fail();
2639 } else {
2640 assert(has_aborted(), "currently the only way to do so");
2641 // The only way to abort the bitmap iteration is to return
2642 // false from the do_bit() method. However, inside the
2643 // do_bit() method we move the _finger to point to the
2644 // object currently being looked at. So, if we bail out, we
2645 // have definitely set _finger to something non-null.
2646 assert(_finger != nullptr, "invariant");
2647
2648 // Region iteration was actually aborted. So now _finger
2649 // points to the address of the object we last scanned. If we
2650 // leave it there, when we restart this task, we will rescan
2651 // the object. It is easy to avoid this. We move the finger by
2652 // enough to point to the next possible object header.
2653 assert(_finger < _region_limit, "invariant");
2654 HeapWord* const new_finger = _finger + cast_to_oop(_finger)->size();
2655 if (new_finger >= _region_limit) {
2656 giveup_current_region();
2657 } else {
2658 move_finger_to(new_finger);
2659 }
2660 }
2661 }
2662
2663 void G1CMTask::claim_new_region() {
2664 // Read the note on the claim_region() method on why it might
2665 // return null with potentially more regions available for
2666 // claiming and why we have to check out_of_regions() to determine
2667 // whether we're done or not.
2668 while (!has_aborted() && _curr_region == nullptr && !_cm->out_of_regions()) {
2669 // We are going to try to claim a new region. We should have
2670 // given up on the previous one.
2671 // Separated the asserts so that we know which one fires.
2672 assert(_curr_region == nullptr, "invariant");
2673 assert(_finger == nullptr, "invariant");
2674 assert(_region_limit == nullptr, "invariant");
2675 G1HeapRegion* claimed_region = _cm->claim_region(_worker_id);
2676 if (claimed_region != nullptr) {
2677 // Yes, we managed to claim one
2678 setup_for_region(claimed_region);
2679 assert(_curr_region == claimed_region, "invariant");
2680 }
2681 // It is important to call the regular clock here. It might take
2682 // a while to claim a region if, for example, we hit a large
2683 // block of empty regions. So we need to call the regular clock
2684 // method once round the loop to make sure it's called
2685 // frequently enough.
2686 abort_marking_if_regular_check_fail();
2687 }
2688 }
2689
2690 void G1CMTask::attempt_stealing() {
2691 // We cannot check whether the global stack is empty, since other
2692 // tasks might be pushing objects to it concurrently.
2693 assert(_cm->out_of_regions() && _task_queue->size() == 0,
2694 "only way to reach here");
2695 while (!has_aborted()) {
2696 G1TaskQueueEntry entry;
2697 if (_cm->try_stealing(_worker_id, entry)) {
2698 process_entry(entry, true /* stolen */);
2699
2700 // And since we're towards the end, let's totally drain the
2701 // local queue and global stack.
2702 drain_local_queue(false);
2703 drain_global_stack(false);
2704 } else {
2705 break;
2706 }
2707 }
2708 }
2709
2710 void G1CMTask::attempt_termination(bool is_serial) {
2711 // We cannot check whether the global stack is empty, since other
2712 // tasks might be concurrently pushing objects on it.
2713 // Separated the asserts so that we know which one fires.
2714 assert(_cm->out_of_regions(), "only way to reach here");
2715 assert(_task_queue->size() == 0, "only way to reach here");
2716 double termination_start_time_ms = os::elapsedTime() * 1000.0;
2717
2718 // The G1CMTask class also extends the TerminatorTerminator class,
2719 // hence its should_exit_termination() method will also decide
2720 // whether to exit the termination protocol or not.
2721 bool finished = (is_serial ||
2722 _cm->terminator()->offer_termination(this));
2723 _termination_time_ms += (os::elapsedTime() * 1000.0 - termination_start_time_ms);
2724
2725 if (finished) {
2726 // We're all done.
2727
2728 // We can now guarantee that the global stack is empty, since
2729 // all other tasks have finished. We separated the guarantees so
2730 // that, if a condition is false, we can immediately find out
2731 // which one.
2732 guarantee(_cm->out_of_regions(), "only way to reach here");
2733 guarantee(_cm->mark_stack_empty(), "only way to reach here");
2734 guarantee(_task_queue->size() == 0, "only way to reach here");
2735 guarantee(!_cm->has_overflown(), "only way to reach here");
2736 guarantee(!has_aborted(), "should never happen if termination has completed");
2737 } else {
2738 // Apparently there's more work to do. Let's abort this task. We
2739 // will restart it and hopefully we can find more things to do.
2740 set_has_aborted();
2741 }
2742 }
2743
2744 void G1CMTask::handle_abort(bool is_serial, double elapsed_time_ms) {
2745 if (_has_timed_out) {
2746 double diff_ms = elapsed_time_ms - _time_target_ms;
2747 // Keep statistics of how well we did with respect to hitting
2748 // our target only if we actually timed out (if we aborted for
2749 // other reasons, then the results might get skewed).
2750 _marking_step_diff_ms.add(diff_ms);
2751 }
2752
2753 if (!_cm->has_overflown()) {
2754 return;
2755 }
2756
2757 // This is the interesting one. We aborted because a global
2758 // overflow was raised. This means we have to restart the
2759 // marking phase and start iterating over regions. However, in
2760 // order to do this we have to make sure that all tasks stop
2761 // what they are doing and re-initialize in a safe manner. We
2762 // will achieve this with the use of two barrier sync points.
2763 if (!is_serial) {
2764 // We only need to enter the sync barrier if being called
2765 // from a parallel context
2766 _cm->enter_first_sync_barrier(_worker_id);
2767
2768 // When we exit this sync barrier we know that all tasks have
2769 // stopped doing marking work. So, it's now safe to
2770 // re-initialize our data structures.
2771 }
2772
2773 clear_region_fields();
2774 flush_mark_stats_cache();
2775
2776 if (!is_serial) {
2777 // If we're executing the concurrent phase of marking, reset the marking
2778 // state; otherwise the marking state is reset after reference processing,
2779 // during the remark pause.
2780 // If we reset here as a result of an overflow during the remark we will
2781 // see assertion failures from any subsequent set_concurrency_and_phase()
2782 // calls.
2783 if (_cm->concurrent() && _worker_id == 0) {
2784 // Worker 0 is responsible for clearing the global data structures because
2785 // of an overflow. During STW we should not clear the overflow flag (in
2786 // G1ConcurrentMark::reset_marking_state()) since we rely on it being true when we exit
2787 // method to abort the pause and restart concurrent marking.
2788 _cm->reset_marking_for_restart();
2789
2790 log_info(gc, marking)("Concurrent Mark reset for overflow");
2791 }
2792
2793 // ...and enter the second barrier.
2794 _cm->enter_second_sync_barrier(_worker_id);
2795 }
2796 }
2797
2798 /*****************************************************************************
2799
2800 The do_marking_step(time_target_ms, ...) method is the building
2801 block of the parallel marking framework. It can be called in parallel
2802 with other invocations of do_marking_step() on different tasks
2803 (but only one per task, obviously) and concurrently with the
2804 mutator threads, or during remark, hence it eliminates the need
2805 for two versions of the code. When called during remark, it will
2806 pick up from where the task left off during the concurrent marking
2807 phase. Interestingly, tasks are also claimable during evacuation
2808 pauses too, since do_marking_step() ensures that it aborts before
2809 it needs to yield.
2810
2811 The data structures that it uses to do marking work are the
2812 following:
2813
2814 (1) Marking Bitmap. If there are grey objects that appear only
2815 on the bitmap (this happens either when dealing with an overflow
2816 or when the concurrent start pause has simply marked the roots
2817 and didn't push them on the stack), then tasks claim heap
2818 regions whose bitmap they then scan to find grey objects. A
2819 global finger indicates where the end of the last claimed region
2820 is. A local finger indicates how far into the region a task has
2821 scanned. The two fingers are used to determine how to grey an
2822 object (i.e. whether simply marking it is OK, as it will be
2823 visited by a task in the future, or whether it needs to be also
2824 pushed on a stack).
2825
2826 (2) Local Queue. The local queue of the task which is accessed
2827 reasonably efficiently by the task. Other tasks can steal from
2828 it when they run out of work. Throughout the marking phase, a
2829 task attempts to keep its local queue short but not totally
2830 empty, so that entries are available for stealing by other
2831 tasks. Only when there is no more work, a task will totally
2832 drain its local queue.
2833
2834 (3) Global Mark Stack. This handles local queue overflow. During
2835 marking only sets of entries are moved between it and the local
2836 queues, as access to it requires a mutex and more fine-grain
2837 interaction with it which might cause contention. If it
2838 overflows, then the marking phase should restart and iterate
2839 over the bitmap to identify grey objects. Throughout the marking
2840 phase, tasks attempt to keep the global mark stack at a small
2841 length but not totally empty, so that entries are available for
2842 popping by other tasks. Only when there is no more work, tasks
2843 will totally drain the global mark stack.
2844
2845 (4) SATB Buffer Queue. This is where completed SATB buffers are
2846 made available. Buffers are regularly removed from this queue
2847 and scanned for roots, so that the queue doesn't get too
2848 long. During remark, all completed buffers are processed, as
2849 well as the filled in parts of any uncompleted buffers.
2850
2851 The do_marking_step() method tries to abort when the time target
2852 has been reached. There are a few other cases when the
2853 do_marking_step() method also aborts:
2854
2855 (1) When the marking phase has been aborted (after a Full GC).
2856
2857 (2) When a global overflow (on the global stack) has been
2858 triggered. Before the task aborts, it will actually sync up with
2859 the other tasks to ensure that all the marking data structures
2860 (local queues, stacks, fingers etc.) are re-initialized so that
2861 when do_marking_step() completes, the marking phase can
2862 immediately restart.
2863
2864 (3) When enough completed SATB buffers are available. The
2865 do_marking_step() method only tries to drain SATB buffers right
2866 at the beginning. So, if enough buffers are available, the
2867 marking step aborts and the SATB buffers are processed at
2868 the beginning of the next invocation.
2869
2870 (4) To yield. when we have to yield then we abort and yield
2871 right at the end of do_marking_step(). This saves us from a lot
2872 of hassle as, by yielding we might allow a Full GC. If this
2873 happens then objects will be compacted underneath our feet, the
2874 heap might shrink, etc. We save checking for this by just
2875 aborting and doing the yield right at the end.
2876
2877 From the above it follows that the do_marking_step() method should
2878 be called in a loop (or, otherwise, regularly) until it completes.
2879
2880 If a marking step completes without its has_aborted() flag being
2881 true, it means it has completed the current marking phase (and
2882 also all other marking tasks have done so and have all synced up).
2883
2884 A method called regular_clock_call() is invoked "regularly" (in
2885 sub ms intervals) throughout marking. It is this clock method that
2886 checks all the abort conditions which were mentioned above and
2887 decides when the task should abort. A work-based scheme is used to
2888 trigger this clock method: when the number of object words the
2889 marking phase has scanned or the number of references the marking
2890 phase has visited reach a given limit. Additional invocations to
2891 the method clock have been planted in a few other strategic places
2892 too. The initial reason for the clock method was to avoid calling
2893 cpu time gathering too regularly, as it is quite expensive. So,
2894 once it was in place, it was natural to piggy-back all the other
2895 conditions on it too and not constantly check them throughout the code.
2896
2897 If do_termination is true then do_marking_step will enter its
2898 termination protocol.
2899
2900 The value of is_serial must be true when do_marking_step is being
2901 called serially (i.e. by the VMThread) and do_marking_step should
2902 skip any synchronization in the termination and overflow code.
2903 Examples include the serial remark code and the serial reference
2904 processing closures.
2905
2906 The value of is_serial must be false when do_marking_step is
2907 being called by any of the worker threads.
2908 Examples include the concurrent marking code (CMMarkingTask),
2909 the MT remark code, and the MT reference processing closures.
2910
2911 *****************************************************************************/
2912
2913 void G1CMTask::do_marking_step(double time_target_ms,
2914 bool do_termination,
2915 bool is_serial) {
2916 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
2917
2918 _start_cpu_time_ns = os::current_thread_cpu_time();
2919
2920 // If do_stealing is true then do_marking_step will attempt to
2921 // steal work from the other G1CMTasks. It only makes sense to
2922 // enable stealing when the termination protocol is enabled
2923 // and do_marking_step() is not being called serially.
2924 bool do_stealing = do_termination && !is_serial;
2925
2926 G1Predictions const& predictor = _g1h->policy()->predictor();
2927 double diff_prediction_ms = predictor.predict_zero_bounded(&_marking_step_diff_ms);
2928 _time_target_ms = time_target_ms - diff_prediction_ms;
2929
2930 // set up the variables that are used in the work-based scheme to
2931 // call the regular clock method
2932 _words_scanned = 0;
2933 _refs_reached = 0;
2934 recalculate_limits();
2935
2936 // clear all flags
2937 clear_has_aborted();
2938 _has_timed_out = false;
2939 _draining_satb_buffers = false;
2940
2941 ++_calls;
2942
2943 // Set up the bitmap and oop closures. Anything that uses them is
2944 // eventually called from this method, so it is OK to allocate these
2945 // statically.
2946 G1CMBitMapClosure bitmap_closure(this, _cm);
2947 G1CMOopClosure cm_oop_closure(_g1h, this);
2948 set_cm_oop_closure(&cm_oop_closure);
2949
2950 if (_cm->has_overflown()) {
2951 // This can happen if the mark stack overflows during a GC pause
2952 // and this task, after a yield point, restarts. We have to abort
2953 // as we need to get into the overflow protocol which happens
2954 // right at the end of this task.
2955 set_has_aborted();
2956 }
2957
2958 // First drain any available SATB buffers. After this, we will not
2959 // look at SATB buffers before the next invocation of this method.
2960 // If enough completed SATB buffers are queued up, the regular clock
2961 // will abort this task so that it restarts.
2962 drain_satb_buffers();
2963 // ...then partially drain the local queue and the global stack
2964 drain_local_queue(true);
2965 drain_global_stack(true);
2966
2967 do {
2968 process_current_region(bitmap_closure);
2969 // At this point we have either completed iterating over the
2970 // region we were holding on to, or we have aborted.
2971
2972 // We then partially drain the local queue and the global stack.
2973 drain_local_queue(true);
2974 drain_global_stack(true);
2975
2976 claim_new_region();
2977
2978 assert(has_aborted() || _curr_region != nullptr || _cm->out_of_regions(),
2979 "at this point we should be out of regions");
2980 } while ( _curr_region != nullptr && !has_aborted());
2981
2982 // We cannot check whether the global stack is empty, since other
2983 // tasks might be pushing objects to it concurrently.
2984 assert(has_aborted() || _cm->out_of_regions(),
2985 "at this point we should be out of regions");
2986 // Try to reduce the number of available SATB buffers so that
2987 // remark has less work to do.
2988 drain_satb_buffers();
2989
2990 // Since we've done everything else, we can now totally drain the
2991 // local queue and global stack.
2992 drain_local_queue(false);
2993 drain_global_stack(false);
2994
2995 // Attempt at work stealing from other task's queues.
2996 if (do_stealing && !has_aborted()) {
2997 // We have not aborted. This means that we have finished all that
2998 // we could. Let's try to do some stealing...
2999 attempt_stealing();
3000 }
3001
3002 // We still haven't aborted. Now, let's try to get into the
3003 // termination protocol.
3004 if (do_termination && !has_aborted()) {
3005 attempt_termination(is_serial);
3006 }
3007
3008 // Mainly for debugging purposes to make sure that a pointer to the
3009 // closure which was statically allocated in this frame doesn't
3010 // escape it by accident.
3011 set_cm_oop_closure(nullptr);
3012 jlong end_cpu_time_ns = os::current_thread_cpu_time();
3013 double elapsed_time_ms = (double)(end_cpu_time_ns - _start_cpu_time_ns) / NANOSECS_PER_MILLISEC;
3014 // Update the step history.
3015 _step_times_ms.add(elapsed_time_ms);
3016
3017 if (has_aborted()) {
3018 // The task was aborted for some reason.
3019 handle_abort(is_serial, elapsed_time_ms);
3020 }
3021 }
3022
3023 G1CMTask::G1CMTask(uint worker_id,
3024 G1ConcurrentMark* cm,
3025 G1CMTaskQueue* task_queue,
3026 G1RegionMarkStats* mark_stats) :
3027 _worker_id(worker_id),
3028 _g1h(G1CollectedHeap::heap()),
3029 _cm(cm),
3030 _mark_bitmap(nullptr),
3031 _task_queue(task_queue),
3032 _partial_array_splitter(_cm->partial_array_state_manager(), _cm->max_num_tasks()),
3033 _mark_stats_cache(mark_stats, G1RegionMarkStatsCache::RegionMarkStatsCacheSize),
3034 _calls(0),
3035 _time_target_ms(0.0),
3036 _start_cpu_time_ns(0),
3037 _cm_oop_closure(nullptr),
3038 _curr_region(nullptr),
3039 _finger(nullptr),
3040 _region_limit(nullptr),
3041 _words_scanned(0),
3042 _words_scanned_limit(0),
3043 _real_words_scanned_limit(0),
3044 _refs_reached(0),
3045 _refs_reached_limit(0),
3046 _real_refs_reached_limit(0),
3047 _has_aborted(false),
3048 _has_timed_out(false),
3049 _draining_satb_buffers(false),
3050 _step_times_ms(),
3051 _elapsed_time_ms(0.0),
3052 _termination_time_ms(0.0),
3053 _marking_step_diff_ms()
3054 {
3055 guarantee(task_queue != nullptr, "invariant");
3056
3057 _marking_step_diff_ms.add(0.5);
3058 }
3059
3060 // These are formatting macros that are used below to ensure
3061 // consistent formatting. The *_H_* versions are used to format the
3062 // header for a particular value and they should be kept consistent
3063 // with the corresponding macro. Also note that most of the macros add
3064 // the necessary white space (as a prefix) which makes them a bit
3065 // easier to compose.
3066
3067 // All the output lines are prefixed with this string to be able to
3068 // identify them easily in a large log file.
3069 #define G1PPRL_LINE_PREFIX "###"
3070
3071 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
3072 #ifdef _LP64
3073 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
3074 #else // _LP64
3075 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
3076 #endif // _LP64
3077
3078 // For per-region info
3079 #define G1PPRL_TYPE_FORMAT " %-4s"
3080 #define G1PPRL_TYPE_H_FORMAT " %4s"
3081 #define G1PPRL_STATE_FORMAT " %-5s"
3082 #define G1PPRL_STATE_H_FORMAT " %5s"
3083 #define G1PPRL_BYTE_FORMAT " %9zu"
3084 #define G1PPRL_BYTE_H_FORMAT " %9s"
3085 #define G1PPRL_DOUBLE_FORMAT "%14.1f"
3086 #define G1PPRL_GCEFF_H_FORMAT " %14s"
3087 #define G1PPRL_GID_H_FORMAT " %9s"
3088 #define G1PPRL_GID_FORMAT " " UINT32_FORMAT_W(9)
3089 #define G1PPRL_LEN_FORMAT " " UINT32_FORMAT_W(14)
3090 #define G1PPRL_LEN_H_FORMAT " %14s"
3091 #define G1PPRL_GID_GCEFF_FORMAT " %14.1f"
3092 #define G1PPRL_GID_LIVENESS_FORMAT " %9.2f"
3093
3094 // For summary info
3095 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
3096 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": %zu"
3097 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
3098 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3099
3100 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) :
3101 _total_used_bytes(0),
3102 _total_capacity_bytes(0),
3103 _total_live_bytes(0),
3104 _total_remset_bytes(0),
3105 _total_code_roots_bytes(0)
3106 {
3107 if (!log_is_enabled(Trace, gc, liveness)) {
3108 return;
3109 }
3110
3111 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3112 MemRegion reserved = g1h->reserved();
3113 double now = os::elapsedTime();
3114
3115 // Print the header of the output.
3116 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3117 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
3118 G1PPRL_SUM_ADDR_FORMAT("reserved")
3119 G1PPRL_SUM_BYTE_FORMAT("region-size"),
3120 p2i(reserved.start()), p2i(reserved.end()),
3121 G1HeapRegion::GrainBytes);
3122 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3123 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3124 G1PPRL_TYPE_H_FORMAT
3125 G1PPRL_ADDR_BASE_H_FORMAT
3126 G1PPRL_BYTE_H_FORMAT
3127 G1PPRL_BYTE_H_FORMAT
3128 G1PPRL_STATE_H_FORMAT
3129 G1PPRL_BYTE_H_FORMAT
3130 G1PPRL_GID_H_FORMAT,
3131 "type", "address-range",
3132 "used", "live",
3133 "state", "code-roots",
3134 "group-id");
3135 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3136 G1PPRL_TYPE_H_FORMAT
3137 G1PPRL_ADDR_BASE_H_FORMAT
3138 G1PPRL_BYTE_H_FORMAT
3139 G1PPRL_BYTE_H_FORMAT
3140 G1PPRL_STATE_H_FORMAT
3141 G1PPRL_BYTE_H_FORMAT
3142 G1PPRL_GID_H_FORMAT,
3143 "", "",
3144 "(bytes)", "(bytes)",
3145 "", "(bytes)", "");
3146 }
3147
3148 bool G1PrintRegionLivenessInfoClosure::do_heap_region(G1HeapRegion* r) {
3149 if (!log_is_enabled(Trace, gc, liveness)) {
3150 return false;
3151 }
3152
3153 const char* type = r->get_type_str();
3154 HeapWord* bottom = r->bottom();
3155 HeapWord* end = r->end();
3156 size_t capacity_bytes = r->capacity();
3157 size_t used_bytes = r->used();
3158 size_t live_bytes = r->live_bytes();
3159 size_t remset_bytes = r->rem_set()->mem_size();
3160 size_t code_roots_bytes = r->rem_set()->code_roots_mem_size();
3161 const char* remset_type = r->rem_set()->get_short_state_str();
3162 uint cset_group_id = r->rem_set()->has_cset_group()
3163 ? r->rem_set()->cset_group_id()
3164 : G1CSetCandidateGroup::NoRemSetId;
3165
3166 _total_used_bytes += used_bytes;
3167 _total_capacity_bytes += capacity_bytes;
3168 _total_live_bytes += live_bytes;
3169 _total_remset_bytes += remset_bytes;
3170 _total_code_roots_bytes += code_roots_bytes;
3171
3172 // Print a line for this particular region.
3173 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3174 G1PPRL_TYPE_FORMAT
3175 G1PPRL_ADDR_BASE_FORMAT
3176 G1PPRL_BYTE_FORMAT
3177 G1PPRL_BYTE_FORMAT
3178 G1PPRL_STATE_FORMAT
3179 G1PPRL_BYTE_FORMAT
3180 G1PPRL_GID_FORMAT,
3181 type, p2i(bottom), p2i(end),
3182 used_bytes, live_bytes,
3183 remset_type, code_roots_bytes,
3184 cset_group_id);
3185
3186 return false;
3187 }
3188
3189 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3190 if (!log_is_enabled(Trace, gc, liveness)) {
3191 return;
3192 }
3193
3194 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3195 _total_remset_bytes += g1h->card_set_freelist_pool()->mem_size();
3196 // add static memory usages to remembered set sizes
3197 _total_remset_bytes += G1HeapRegionRemSet::static_mem_size();
3198
3199 log_cset_candidate_groups();
3200
3201 // Print the footer of the output.
3202 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3203 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3204 " SUMMARY"
3205 G1PPRL_SUM_MB_FORMAT("capacity")
3206 G1PPRL_SUM_MB_PERC_FORMAT("used")
3207 G1PPRL_SUM_MB_PERC_FORMAT("live")
3208 G1PPRL_SUM_MB_FORMAT("remset")
3209 G1PPRL_SUM_MB_FORMAT("code-roots"),
3210 bytes_to_mb(_total_capacity_bytes),
3211 bytes_to_mb(_total_used_bytes),
3212 percent_of(_total_used_bytes, _total_capacity_bytes),
3213 bytes_to_mb(_total_live_bytes),
3214 percent_of(_total_live_bytes, _total_capacity_bytes),
3215 bytes_to_mb(_total_remset_bytes),
3216 bytes_to_mb(_total_code_roots_bytes));
3217 }
3218
3219 void G1PrintRegionLivenessInfoClosure::log_cset_candidate_group_add_total(G1CSetCandidateGroup* group, const char* type) {
3220 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3221 G1PPRL_GID_FORMAT
3222 G1PPRL_LEN_FORMAT
3223 G1PPRL_GID_GCEFF_FORMAT
3224 G1PPRL_GID_LIVENESS_FORMAT
3225 G1PPRL_BYTE_FORMAT
3226 G1PPRL_TYPE_H_FORMAT,
3227 group->group_id(),
3228 group->length(),
3229 group->length() > 0 ? group->gc_efficiency() : 0.0,
3230 group->length() > 0 ? group->liveness_percent() : 0.0,
3231 group->card_set()->mem_size(),
3232 type);
3233 _total_remset_bytes += group->card_set()->mem_size();
3234 }
3235
3236 void G1PrintRegionLivenessInfoClosure::log_cset_candidate_grouplist(G1CSetCandidateGroupList& gl, const char* type) {
3237 for (G1CSetCandidateGroup* group : gl) {
3238 log_cset_candidate_group_add_total(group, type);
3239 }
3240 }
3241
3242 void G1PrintRegionLivenessInfoClosure::log_cset_candidate_groups() {
3243 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3244 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" Collection Set Candidate Groups");
3245 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX " Types: Y=Young, M=From Marking Regions, R=Retained Regions");
3246 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3247 G1PPRL_GID_H_FORMAT
3248 G1PPRL_LEN_H_FORMAT
3249 G1PPRL_GCEFF_H_FORMAT
3250 G1PPRL_BYTE_H_FORMAT
3251 G1PPRL_BYTE_H_FORMAT
3252 G1PPRL_TYPE_H_FORMAT,
3253 "groud-id", "num-regions",
3254 "gc-eff", "liveness",
3255 "remset", "type");
3256
3257 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3258 G1PPRL_GID_H_FORMAT
3259 G1PPRL_LEN_H_FORMAT
3260 G1PPRL_GCEFF_H_FORMAT
3261 G1PPRL_BYTE_H_FORMAT
3262 G1PPRL_BYTE_H_FORMAT
3263 G1PPRL_TYPE_H_FORMAT,
3264 "", "",
3265 "(bytes/ms)", "%",
3266 "(bytes)", "");
3267
3268 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3269
3270 log_cset_candidate_group_add_total(g1h->young_regions_cset_group(), "Y");
3271
3272 G1CollectionSetCandidates* candidates = g1h->policy()->candidates();
3273 log_cset_candidate_grouplist(candidates->from_marking_groups(), "M");
3274 log_cset_candidate_grouplist(candidates->retained_groups(), "R");
3275 }