1 /*
2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25
26 #include "precompiled.hpp"
27
28 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahAsserts.hpp"
30 #include "gc/shenandoah/shenandoahCardTable.hpp"
31 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
38 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
39 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
40 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
41 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
42 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
43 #include "gc/shenandoah/shenandoahUtils.hpp"
44 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
45 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
46 #include "runtime/threads.hpp"
47 #include "utilities/events.hpp"
48
49 class ShenandoahFlushAllSATB : public ThreadClosure {
50 private:
51 SATBMarkQueueSet& _satb_qset;
52
53 public:
54 explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) :
55 _satb_qset(satb_qset) {}
56
57 void do_thread(Thread* thread) override {
58 // Transfer any partial buffer to the qset for completed buffer processing.
59 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
60 }
61 };
62
63 class ShenandoahProcessOldSATB : public SATBBufferClosure {
64 private:
65 ShenandoahObjToScanQueue* _queue;
66 ShenandoahHeap* _heap;
67 ShenandoahMarkingContext* const _mark_context;
68 size_t _trashed_oops;
69
70 public:
71 explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) :
72 _queue(q),
73 _heap(ShenandoahHeap::heap()),
74 _mark_context(_heap->marking_context()),
75 _trashed_oops(0) {}
76
77 void do_buffer(void** buffer, size_t size) override {
78 assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here");
79 for (size_t i = 0; i < size; ++i) {
80 oop *p = (oop *) &buffer[i];
81 ShenandoahHeapRegion* region = _heap->heap_region_containing(*p);
82 if (region->is_old() && region->is_active()) {
83 ShenandoahMark::mark_through_ref<oop, OLD>(p, _queue, nullptr, _mark_context, false);
84 } else {
85 _trashed_oops++;
86 }
87 }
88 }
89
90 size_t trashed_oops() const {
91 return _trashed_oops;
92 }
93 };
94
95 class ShenandoahPurgeSATBTask : public WorkerTask {
96 private:
97 ShenandoahObjToScanQueueSet* _mark_queues;
98 // Keep track of the number of oops that are not transferred to mark queues.
99 // This is volatile because workers update it, but the vm thread reads it.
100 volatile size_t _trashed_oops;
101
102 public:
103 explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) :
104 WorkerTask("Purge SATB"),
105 _mark_queues(queues),
106 _trashed_oops(0) {
107 Threads::change_thread_claim_token();
108 }
109
110 ~ShenandoahPurgeSATBTask() {
111 if (_trashed_oops > 0) {
112 log_debug(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers", _trashed_oops);
113 }
114 }
115
116 void work(uint worker_id) override {
117 ShenandoahParallelWorkerSession worker_session(worker_id);
118 ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
119 ShenandoahFlushAllSATB flusher(satb_queues);
120 Threads::possibly_parallel_threads_do(true /* is_par */, &flusher);
121
122 ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id);
123 ShenandoahProcessOldSATB processor(mark_queue);
124 while (satb_queues.apply_closure_to_completed_buffer(&processor)) {}
125
126 Atomic::add(&_trashed_oops, processor.trashed_oops());
127 }
128 };
129
130 class ShenandoahTransferOldSATBTask : public WorkerTask {
131 ShenandoahSATBMarkQueueSet& _satb_queues;
132 ShenandoahObjToScanQueueSet* _mark_queues;
133 // Keep track of the number of oops that are not transferred to mark queues.
134 // This is volatile because workers update it, but the control thread reads it.
135 volatile size_t _trashed_oops;
136
137 public:
138 explicit ShenandoahTransferOldSATBTask(ShenandoahSATBMarkQueueSet& satb_queues, ShenandoahObjToScanQueueSet* mark_queues) :
139 WorkerTask("Transfer SATB"),
140 _satb_queues(satb_queues),
141 _mark_queues(mark_queues),
142 _trashed_oops(0) {}
143
144 ~ShenandoahTransferOldSATBTask() {
145 if (_trashed_oops > 0) {
146 log_debug(gc)("Purged %zu oops from old generation SATB buffers", _trashed_oops);
147 }
148 }
149
150 void work(uint worker_id) override {
151 ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id);
152 ShenandoahProcessOldSATB processor(mark_queue);
153 while (_satb_queues.apply_closure_to_completed_buffer(&processor)) {}
154
155 Atomic::add(&_trashed_oops, processor.trashed_oops());
156 }
157 };
158
159 class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask {
160 private:
161 uint _nworkers;
162 ShenandoahHeapRegion** _coalesce_and_fill_region_array;
163 uint _coalesce_and_fill_region_count;
164 volatile bool _is_preempted;
165
166 public:
167 ShenandoahConcurrentCoalesceAndFillTask(uint nworkers,
168 ShenandoahHeapRegion** coalesce_and_fill_region_array,
169 uint region_count) :
170 WorkerTask("Shenandoah Concurrent Coalesce and Fill"),
171 _nworkers(nworkers),
172 _coalesce_and_fill_region_array(coalesce_and_fill_region_array),
173 _coalesce_and_fill_region_count(region_count),
174 _is_preempted(false) {
175 }
176
177 void work(uint worker_id) override {
178 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_coalesce_and_fill, ShenandoahPhaseTimings::ScanClusters, worker_id);
179 for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) {
180 ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx];
181 if (r->is_humongous()) {
182 // There is only one object in this region and it is not garbage,
183 // so no need to coalesce or fill.
184 continue;
185 }
186
187 if (!r->oop_coalesce_and_fill(true)) {
188 // Coalesce and fill has been preempted
189 Atomic::store(&_is_preempted, true);
190 return;
191 }
192 }
193 }
194
195 // Value returned from is_completed() is only valid after all worker thread have terminated.
196 bool is_completed() {
197 return !Atomic::load(&_is_preempted);
198 }
199 };
200
201 ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity)
202 : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity),
203 _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)),
204 _old_heuristics(nullptr),
205 _region_balance(0),
206 _promoted_reserve(0),
207 _promoted_expended(0),
208 _promotion_potential(0),
209 _pad_for_promote_in_place(0),
210 _promotable_humongous_regions(0),
211 _promotable_regular_regions(0),
212 _is_parsable(true),
213 _card_scan(nullptr),
214 _state(WAITING_FOR_BOOTSTRAP),
215 _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION),
216 _min_growth_before_compaction ((ShenandoahMinOldGenGrowthPercent * FRACTIONAL_DENOMINATOR) / 100)
217 {
218 _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
219 // Always clear references for old generation
220 ref_processor()->set_soft_reference_policy(true);
221
222 if (ShenandoahCardBarrier) {
223 ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table();
224 size_t card_count = card_table->cards_required(ShenandoahHeap::heap()->reserved_region().word_size());
225 auto rs = new ShenandoahDirectCardMarkRememberedSet(card_table, card_count);
226 _card_scan = new ShenandoahScanRemembered(rs);
227 }
228 }
229
230 void ShenandoahOldGeneration::set_promoted_reserve(size_t new_val) {
231 shenandoah_assert_heaplocked_or_safepoint();
232 _promoted_reserve = new_val;
233 }
234
235 size_t ShenandoahOldGeneration::get_promoted_reserve() const {
236 return _promoted_reserve;
237 }
238
239 void ShenandoahOldGeneration::augment_promoted_reserve(size_t increment) {
240 shenandoah_assert_heaplocked_or_safepoint();
241 _promoted_reserve += increment;
242 }
243
244 void ShenandoahOldGeneration::reset_promoted_expended() {
245 shenandoah_assert_heaplocked_or_safepoint();
246 Atomic::store(&_promoted_expended, (size_t) 0);
247 }
248
249 size_t ShenandoahOldGeneration::expend_promoted(size_t increment) {
250 shenandoah_assert_heaplocked_or_safepoint();
251 assert(get_promoted_expended() + increment <= get_promoted_reserve(), "Do not expend more promotion than budgeted");
252 return Atomic::add(&_promoted_expended, increment);
253 }
254
255 size_t ShenandoahOldGeneration::unexpend_promoted(size_t decrement) {
256 return Atomic::sub(&_promoted_expended, decrement);
257 }
258
259 size_t ShenandoahOldGeneration::get_promoted_expended() const {
260 return Atomic::load(&_promoted_expended);
261 }
262
263 bool ShenandoahOldGeneration::can_allocate(const ShenandoahAllocRequest &req) const {
264 assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory");
265
266 const size_t requested_bytes = req.size() * HeapWordSize;
267 // The promotion reserve may also be used for evacuations. If we can promote this object,
268 // then we can also evacuate it.
269 if (can_promote(requested_bytes)) {
270 // The promotion reserve should be able to accommodate this request. The request
271 // might still fail if alignment with the card table increases the size. The request
272 // may also fail if the heap is badly fragmented and the free set cannot find room for it.
273 return true;
274 }
275
276 if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
277 // The promotion reserve cannot accommodate this plab request. Check if we still have room for
278 // evacuations. Note that we cannot really know how much of the plab will be used for evacuations,
279 // so here we only check that some evacuation reserve still exists.
280 return get_evacuation_reserve() > 0;
281 }
282
283 // This is a shared allocation request. We've already checked that it can't be promoted, so if
284 // it is a promotion, we return false. Otherwise, it is a shared evacuation request, and we allow
285 // the allocation to proceed.
286 return !req.is_promotion();
287 }
288
289 void
290 ShenandoahOldGeneration::configure_plab_for_current_thread(const ShenandoahAllocRequest &req) {
291 // Note: Even when a mutator is performing a promotion outside a LAB, we use a 'shared_gc' request.
292 if (req.is_gc_alloc()) {
293 const size_t actual_size = req.actual_size() * HeapWordSize;
294 if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
295 // We've created a new plab. Now we configure it whether it will be used for promotions
296 // and evacuations - or just evacuations.
297 Thread* thread = Thread::current();
298 ShenandoahThreadLocalData::reset_plab_promoted(thread);
299
300 // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries).
301 // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread.
302 if (can_promote(actual_size)) {
303 // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach.
304 // When we retire this plab, we'll unexpend what we don't really use.
305 expend_promoted(actual_size);
306 ShenandoahThreadLocalData::enable_plab_promotions(thread);
307 ShenandoahThreadLocalData::set_plab_actual_size(thread, actual_size);
308 } else {
309 // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations.
310 ShenandoahThreadLocalData::disable_plab_promotions(thread);
311 ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
312 }
313 } else if (req.is_promotion()) {
314 // Shared promotion.
315 expend_promoted(actual_size);
316 }
317 }
318 }
319
320 size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const {
321 return _live_bytes_after_last_mark;
322 }
323
324 void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) {
325 if (bytes == 0) {
326 // Restart search for best old-gen size to the initial state
327 _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
328 _growth_before_compaction = INITIAL_GROWTH_BEFORE_COMPACTION;
329 } else {
330 _live_bytes_after_last_mark = bytes;
331 _growth_before_compaction /= 2;
332 if (_growth_before_compaction < _min_growth_before_compaction) {
333 _growth_before_compaction = _min_growth_before_compaction;
334 }
335 }
336 }
337
338 void ShenandoahOldGeneration::handle_failed_transfer() {
339 _old_heuristics->trigger_cannot_expand();
340 }
341
342 size_t ShenandoahOldGeneration::usage_trigger_threshold() const {
343 size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR;
344 return result;
345 }
346
347 bool ShenandoahOldGeneration::contains(ShenandoahAffiliation affiliation) const {
348 return affiliation == OLD_GENERATION;
349 }
350 bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const {
351 return region->is_old();
352 }
353
354 void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
355 ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
356 ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions_cl);
357 }
358
359 void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
360 ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
361 ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl);
362 }
363
364 void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) {
365 ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress);
366 }
367
368 bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() {
369 return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress();
370 }
371
372 void ShenandoahOldGeneration::cancel_marking() {
373 if (is_concurrent_mark_in_progress()) {
374 log_debug(gc)("Abandon SATB buffers");
375 ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
376 }
377
378 ShenandoahGeneration::cancel_marking();
379 }
380
381 void ShenandoahOldGeneration::cancel_gc() {
382 shenandoah_assert_safepoint();
383 if (is_idle()) {
384 #ifdef ASSERT
385 validate_waiting_for_bootstrap();
386 #endif
387 } else {
388 log_info(gc)("Terminating old gc cycle.");
389 // Stop marking
390 cancel_marking();
391 // Stop tracking old regions
392 abandon_collection_candidates();
393 // Remove old generation access to young generation mark queues
394 ShenandoahHeap::heap()->young_generation()->set_old_gen_task_queues(nullptr);
395 // Transition to IDLE now.
396 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
397 }
398 }
399
400 void ShenandoahOldGeneration::prepare_gc() {
401 // Now that we have made the old generation parsable, it is safe to reset the mark bitmap.
402 assert(state() != FILLING, "Cannot reset old without making it parsable");
403
404 ShenandoahGeneration::prepare_gc();
405 }
406
407 bool ShenandoahOldGeneration::entry_coalesce_and_fill() {
408 ShenandoahHeap* const heap = ShenandoahHeap::heap();
409
410 static const char* msg = "Coalescing and filling (Old)";
411 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
412
413 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
414 EventMark em("%s", msg);
415 ShenandoahWorkerScope scope(heap->workers(),
416 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
417 msg);
418
419 return coalesce_and_fill();
420 }
421
422 // Make the old generation regions parsable, so they can be safely
423 // scanned when looking for objects in memory indicated by dirty cards.
424 bool ShenandoahOldGeneration::coalesce_and_fill() {
425 transition_to(FILLING);
426
427 // This code will see the same set of regions to fill on each resumption as it did
428 // on the initial run. That's okay because each region keeps track of its own coalesce
429 // and fill state. Regions that were filled on a prior attempt will not try to fill again.
430 uint coalesce_and_fill_regions_count = _old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array);
431 assert(coalesce_and_fill_regions_count <= ShenandoahHeap::heap()->num_regions(), "Sanity");
432 if (coalesce_and_fill_regions_count == 0) {
433 // No regions need to be filled.
434 abandon_collection_candidates();
435 return true;
436 }
437
438 ShenandoahHeap* const heap = ShenandoahHeap::heap();
439 WorkerThreads* workers = heap->workers();
440 uint nworkers = workers->active_workers();
441 ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count);
442
443 log_debug(gc)("Starting (or resuming) coalesce-and-fill of " UINT32_FORMAT " old heap regions", coalesce_and_fill_regions_count);
444 workers->run_task(&task);
445 if (task.is_completed()) {
446 // We no longer need to track regions that need to be coalesced and filled.
447 abandon_collection_candidates();
448 return true;
449 } else {
450 // Coalesce-and-fill has been preempted. We'll finish that effort in the future. Do not invoke
451 // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap
452 // and invokes set_mark_incomplete(). Coalesce-and-fill depends on the mark bitmap.
453 log_debug(gc)("Suspending coalesce-and-fill of old heap regions");
454 return false;
455 }
456 }
457
458 void ShenandoahOldGeneration::concurrent_transfer_pointers_from_satb() const {
459 const ShenandoahHeap* heap = ShenandoahHeap::heap();
460 assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking.");
461 log_debug(gc)("Transfer SATB buffers");
462
463 // Step 1. All threads need to 'complete' partially filled, thread local SATB buffers. This
464 // is accomplished in ShenandoahConcurrentGC::complete_abbreviated_cycle using a Handshake
465 // operation.
466 // Step 2. Use worker threads to transfer oops from old, active regions in the completed
467 // SATB buffers to old generation mark queues.
468 ShenandoahSATBMarkQueueSet& satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
469 ShenandoahTransferOldSATBTask transfer_task(satb_queues, task_queues());
470 heap->workers()->run_task(&transfer_task);
471 }
472
473 void ShenandoahOldGeneration::transfer_pointers_from_satb() const {
474 const ShenandoahHeap* heap = ShenandoahHeap::heap();
475 assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking.");
476 log_debug(gc)("Transfer SATB buffers");
477 ShenandoahPurgeSATBTask purge_satb_task(task_queues());
478 heap->workers()->run_task(&purge_satb_task);
479 }
480
481 bool ShenandoahOldGeneration::contains(oop obj) const {
482 return ShenandoahHeap::heap()->is_in_old(obj);
483 }
484
485 void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) {
486 ShenandoahHeap* heap = ShenandoahHeap::heap();
487 assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
488
489 {
490 ShenandoahGCPhase phase(concurrent ?
491 ShenandoahPhaseTimings::final_update_region_states :
492 ShenandoahPhaseTimings::degen_gc_final_update_region_states);
493 ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
494
495 parallel_heap_region_iterate(&cl);
496 heap->assert_pinned_region_status();
497 }
498
499 {
500 // This doesn't actually choose a collection set, but prepares a list of
501 // regions as 'candidates' for inclusion in a mixed collection.
502 ShenandoahGCPhase phase(concurrent ?
503 ShenandoahPhaseTimings::choose_cset :
504 ShenandoahPhaseTimings::degen_gc_choose_cset);
505 ShenandoahHeapLocker locker(heap->lock());
506 _old_heuristics->prepare_for_old_collections();
507 }
508
509 {
510 // Though we did not choose a collection set above, we still may have
511 // freed up immediate garbage regions so proceed with rebuilding the free set.
512 ShenandoahGCPhase phase(concurrent ?
513 ShenandoahPhaseTimings::final_rebuild_freeset :
514 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
515 ShenandoahHeapLocker locker(heap->lock());
516 size_t cset_young_regions, cset_old_regions;
517 size_t first_old, last_old, num_old;
518 heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old);
519 // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here
520 // is in case there was any immediate old garbage identified.
521 heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old);
522 }
523 }
524
525 const char* ShenandoahOldGeneration::state_name(State state) {
526 switch (state) {
527 case WAITING_FOR_BOOTSTRAP: return "Waiting for Bootstrap";
528 case FILLING: return "Coalescing";
529 case BOOTSTRAPPING: return "Bootstrapping";
530 case MARKING: return "Marking";
531 case EVACUATING: return "Evacuating";
532 case EVACUATING_AFTER_GLOBAL: return "Evacuating (G)";
533 default:
534 ShouldNotReachHere();
535 return "Unknown";
536 }
537 }
538
539 void ShenandoahOldGeneration::transition_to(State new_state) {
540 if (_state != new_state) {
541 log_debug(gc, thread)("Old generation transition from %s to %s", state_name(_state), state_name(new_state));
542 EventMark event("Old was %s, now is %s", state_name(_state), state_name(new_state));
543 validate_transition(new_state);
544 _state = new_state;
545 }
546 }
547
548 #ifdef ASSERT
549 // This diagram depicts the expected state transitions for marking the old generation
550 // and preparing for old collections. When a young generation cycle executes, the
551 // remembered set scan must visit objects in old regions. Visiting an object which
552 // has become dead on previous old cycles will result in crashes. To avoid visiting
553 // such objects, the remembered set scan will use the old generation mark bitmap when
554 // possible. It is _not_ possible to use the old generation bitmap when old marking
555 // is active (bitmap is not complete). For this reason, the old regions are made
556 // parsable _before_ the old generation bitmap is reset. The diagram does not depict
557 // cancellation of old collections by global or full collections.
558 //
559 // When a global collection supersedes an old collection, the global mark still
560 // "completes" the old mark bitmap. Subsequent remembered set scans may use the
561 // old generation mark bitmap, but any uncollected old regions must still be made parsable
562 // before the next old generation cycle begins. For this reason, a global collection may
563 // create mixed collection candidates and coalesce and fill candidates and will put
564 // the old generation in the respective states (EVACUATING or FILLING). After a Full GC,
565 // the mark bitmaps are all reset, all regions are parsable and the mark context will
566 // not be "complete". After a Full GC, remembered set scans will _not_ use the mark bitmap
567 // and we expect the old generation to be waiting for bootstrap.
568 //
569 // +-----------------+
570 // +------------> | FILLING | <---+
571 // | +--------> | | |
572 // | | +-----------------+ |
573 // | | | |
574 // | | | Filling Complete | <-> A global collection may
575 // | | v | move the old generation
576 // | | +-----------------+ | directly from waiting for
577 // +-- |-- |--------> | WAITING | | bootstrap to filling or
578 // | | | +---- | FOR BOOTSTRAP | ----+ evacuating. It may also
579 // | | | | +-----------------+ move from filling to waiting
580 // | | | | | for bootstrap.
581 // | | | | | Reset Bitmap
582 // | | | | v
583 // | | | | +-----------------+ +----------------------+
584 // | | | | | BOOTSTRAP | <-> | YOUNG GC |
585 // | | | | | | | (RSet Parses Region) |
586 // | | | | +-----------------+ +----------------------+
587 // | | | | |
588 // | | | | | Old Marking
589 // | | | | v
590 // | | | | +-----------------+ +----------------------+
591 // | | | | | MARKING | <-> | YOUNG GC |
592 // | | +--------- | | | (RSet Parses Region) |
593 // | | | +-----------------+ +----------------------+
594 // | | | |
595 // | | | | Has Evacuation Candidates
596 // | | | v
597 // | | | +-----------------+ +--------------------+
598 // | | +---> | EVACUATING | <-> | YOUNG GC |
599 // | +------------- | | | (RSet Uses Bitmap) |
600 // | +-----------------+ +--------------------+
601 // | |
602 // | | Global Cycle Coalesces and Fills Old Regions
603 // | v
604 // | +-----------------+ +--------------------+
605 // +----------------- | EVACUATING | <-> | YOUNG GC |
606 // | AFTER GLOBAL | | (RSet Uses Bitmap) |
607 // +-----------------+ +--------------------+
608 //
609 //
610 void ShenandoahOldGeneration::validate_transition(State new_state) {
611 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
612 switch (new_state) {
613 case FILLING:
614 assert(_state != BOOTSTRAPPING, "Cannot begin making old regions parsable after bootstrapping");
615 assert(is_mark_complete(), "Cannot begin filling without first completing marking, state is '%s'", state_name(_state));
616 assert(_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot begin filling without something to fill.");
617 break;
618 case WAITING_FOR_BOOTSTRAP:
619 // GC cancellation can send us back here from any state.
620 validate_waiting_for_bootstrap();
621 break;
622 case BOOTSTRAPPING:
623 assert(_state == WAITING_FOR_BOOTSTRAP, "Cannot reset bitmap without making old regions parsable, state is '%s'", state_name(_state));
624 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates");
625 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parsable.");
626 break;
627 case MARKING:
628 assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state));
629 assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues.");
630 assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now.");
631 break;
632 case EVACUATING_AFTER_GLOBAL:
633 assert(_state == EVACUATING, "Must have been evacuating, state is '%s'", state_name(_state));
634 break;
635 case EVACUATING:
636 assert(_state == WAITING_FOR_BOOTSTRAP || _state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state));
637 assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here.");
638 break;
639 default:
640 fatal("Unknown new state");
641 }
642 }
643
644 bool ShenandoahOldGeneration::validate_waiting_for_bootstrap() {
645 ShenandoahHeap* heap = ShenandoahHeap::heap();
646 assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become ready for bootstrap during old mark.");
647 assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become ready for bootstrap when still setup for bootstrapping.");
648 assert(!is_concurrent_mark_in_progress(), "Cannot be marking in IDLE");
649 assert(!heap->young_generation()->is_bootstrap_cycle(), "Cannot have old mark queues if IDLE");
650 assert(!_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot have coalesce and fill candidates in IDLE");
651 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot have mixed collection candidates in IDLE");
652 return true;
653 }
654 #endif
655
656 ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
657 _old_heuristics = new ShenandoahOldHeuristics(this, ShenandoahGenerationalHeap::heap());
658 _old_heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval);
659 _heuristics = _old_heuristics;
660 return _heuristics;
661 }
662
663 void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) {
664 heuristics()->record_success_concurrent();
665 ShenandoahHeap::heap()->shenandoah_policy()->record_success_old();
666 }
667
668 void ShenandoahOldGeneration::handle_failed_evacuation() {
669 if (_failed_evacuation.try_set()) {
670 log_debug(gc)("Old gen evac failure.");
671 }
672 }
673
674 void ShenandoahOldGeneration::handle_failed_promotion(Thread* thread, size_t size) {
675 // We squelch excessive reports to reduce noise in logs.
676 const size_t MaxReportsPerEpoch = 4;
677 static size_t last_report_epoch = 0;
678 static size_t epoch_report_count = 0;
679 auto heap = ShenandoahGenerationalHeap::heap();
680
681 size_t promotion_reserve;
682 size_t promotion_expended;
683
684 const size_t gc_id = heap->control_thread()->get_gc_id();
685
686 if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) {
687 {
688 // Promotion failures should be very rare. Invest in providing useful diagnostic info.
689 ShenandoahHeapLocker locker(heap->lock());
690 promotion_reserve = get_promoted_reserve();
691 promotion_expended = get_promoted_expended();
692 }
693 PLAB* const plab = ShenandoahThreadLocalData::plab(thread);
694 const size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining();
695 const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled";
696
697 log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT
698 ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT
699 ", old capacity: " SIZE_FORMAT ", old_used: " SIZE_FORMAT ", old unaffiliated regions: " SIZE_FORMAT,
700 size * HeapWordSize, plab == nullptr? "no": "yes",
701 words_remaining * HeapWordSize, promote_enabled, promotion_reserve, promotion_expended,
702 max_capacity(), used(), free_unaffiliated_regions());
703
704 if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) {
705 log_debug(gc, ergo)("Squelching additional promotion failure reports for current epoch");
706 } else if (gc_id != last_report_epoch) {
707 last_report_epoch = gc_id;
708 epoch_report_count = 1;
709 }
710 }
711 }
712
713 void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, bool promotion) {
714 // Only register the copy of the object that won the evacuation race.
715 _card_scan->register_object_without_lock(obj);
716
717 // Mark the entire range of the evacuated object as dirty. At next remembered set scan,
718 // we will clear dirty bits that do not hold interesting pointers. It's more efficient to
719 // do this in batch, in a background GC thread than to try to carefully dirty only cards
720 // that hold interesting pointers right now.
721 _card_scan->mark_range_as_dirty(obj, words);
722
723 if (promotion) {
724 // This evacuation was a promotion, track this as allocation against old gen
725 increase_allocated(words * HeapWordSize);
726 }
727 }
728
729 bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() {
730 return _old_heuristics->unprocessed_old_collection_candidates() > 0;
731 }
732
733 size_t ShenandoahOldGeneration::unprocessed_collection_candidates_live_memory() {
734 return _old_heuristics->unprocessed_old_collection_candidates_live_memory();
735 }
736
737 void ShenandoahOldGeneration::abandon_collection_candidates() {
738 _old_heuristics->abandon_collection_candidates();
739 }
740
741 void ShenandoahOldGeneration::prepare_for_mixed_collections_after_global_gc() {
742 assert(is_mark_complete(), "Expected old generation mark to be complete after global cycle.");
743 _old_heuristics->prepare_for_old_collections();
744 log_info(gc, ergo)("After choosing global collection set, mixed candidates: " UINT32_FORMAT ", coalescing candidates: " SIZE_FORMAT,
745 _old_heuristics->unprocessed_old_collection_candidates(),
746 _old_heuristics->coalesce_and_fill_candidates_count());
747 }
748
749 void ShenandoahOldGeneration::parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl) {
750 // Iterate over old and free regions (exclude young).
751 ShenandoahExcludeRegionClosure<YOUNG_GENERATION> exclude_cl(cl);
752 ShenandoahGeneration::parallel_heap_region_iterate_free(&exclude_cl);
753 }
754
755 void ShenandoahOldGeneration::set_parsable(bool parsable) {
756 _is_parsable = parsable;
757 if (_is_parsable) {
758 // The current state would have been chosen during final mark of the global
759 // collection, _before_ any decisions about class unloading have been made.
760 //
761 // After unloading classes, we have made the old generation regions parsable.
762 // We can skip filling or transition to a state that knows everything has
763 // already been filled.
764 switch (state()) {
765 case ShenandoahOldGeneration::EVACUATING:
766 transition_to(ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL);
767 break;
768 case ShenandoahOldGeneration::FILLING:
769 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Expected no mixed collection candidates");
770 assert(_old_heuristics->coalesce_and_fill_candidates_count() > 0, "Expected coalesce and fill candidates");
771 // When the heuristic put the old generation in this state, it didn't know
772 // that we would unload classes and make everything parsable. But, we know
773 // that now so we can override this state.
774 abandon_collection_candidates();
775 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
776 break;
777 default:
778 // We can get here during a full GC. The full GC will cancel anything
779 // happening in the old generation and return it to the waiting for bootstrap
780 // state. The full GC will then record that the old regions are parsable
781 // after rebuilding the remembered set.
782 assert(is_idle(), "Unexpected state %s at end of global GC", state_name());
783 break;
784 }
785 }
786 }
787
788 void ShenandoahOldGeneration::complete_mixed_evacuations() {
789 assert(is_doing_mixed_evacuations(), "Mixed evacuations should be in progress");
790 if (!_old_heuristics->has_coalesce_and_fill_candidates()) {
791 // No candidate regions to coalesce and fill
792 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
793 return;
794 }
795
796 if (state() == ShenandoahOldGeneration::EVACUATING) {
797 transition_to(ShenandoahOldGeneration::FILLING);
798 return;
799 }
800
801 // Here, we have no more candidates for mixed collections. The candidates for coalescing
802 // and filling have already been processed during the global cycle, so there is nothing
803 // more to do.
804 assert(state() == ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL, "Should be evacuating after a global cycle");
805 abandon_collection_candidates();
806 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
807 }
808
809 void ShenandoahOldGeneration::abandon_mixed_evacuations() {
810 switch(state()) {
811 case ShenandoahOldGeneration::EVACUATING:
812 transition_to(ShenandoahOldGeneration::FILLING);
813 break;
814 case ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL:
815 abandon_collection_candidates();
816 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
817 break;
818 default:
819 log_warning(gc)("Abandon mixed evacuations in unexpected state: %s", state_name(state()));
820 ShouldNotReachHere();
821 break;
822 }
823 }
824
825 void ShenandoahOldGeneration::clear_cards_for(ShenandoahHeapRegion* region) {
826 _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom()));
827 }
828
829 void ShenandoahOldGeneration::mark_card_as_dirty(void* location) {
830 _card_scan->mark_card_as_dirty((HeapWord*)location);
831 }