1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 26 #include "precompiled.hpp" 27 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" 30 #include "gc/shenandoah/shenandoahAsserts.hpp" 31 #include "gc/shenandoah/shenandoahCardTable.hpp" 32 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 33 #include "gc/shenandoah/shenandoahFreeSet.hpp" 34 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 35 #include "gc/shenandoah/shenandoahHeap.hpp" 36 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 37 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 38 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp" 39 #include "gc/shenandoah/shenandoahMarkClosures.hpp" 40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 41 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 42 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 45 #include "gc/shenandoah/shenandoahUtils.hpp" 46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 47 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 48 #include "runtime/threads.hpp" 49 #include "utilities/events.hpp" 50 51 class ShenandoahFlushAllSATB : public ThreadClosure { 52 private: 53 SATBMarkQueueSet& _satb_qset; 54 55 public: 56 explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : 57 _satb_qset(satb_qset) {} 58 59 void do_thread(Thread* thread) override { 60 // Transfer any partial buffer to the qset for completed buffer processing. 61 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 62 } 63 }; 64 65 class ShenandoahProcessOldSATB : public SATBBufferClosure { 66 private: 67 ShenandoahObjToScanQueue* _queue; 68 ShenandoahHeap* _heap; 69 ShenandoahMarkingContext* const _mark_context; 70 size_t _trashed_oops; 71 72 public: 73 explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) : 74 _queue(q), 75 _heap(ShenandoahHeap::heap()), 76 _mark_context(_heap->marking_context()), 77 _trashed_oops(0) {} 78 79 void do_buffer(void** buffer, size_t size) override { 80 assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); 81 for (size_t i = 0; i < size; ++i) { 82 oop *p = (oop *) &buffer[i]; 83 ShenandoahHeapRegion* region = _heap->heap_region_containing(*p); 84 if (region->is_old() && region->is_active()) { 85 ShenandoahMark::mark_through_ref<oop, OLD>(p, _queue, nullptr, _mark_context, false); 86 } else { 87 _trashed_oops++; 88 } 89 } 90 } 91 92 size_t trashed_oops() const { 93 return _trashed_oops; 94 } 95 }; 96 97 class ShenandoahPurgeSATBTask : public WorkerTask { 98 private: 99 ShenandoahObjToScanQueueSet* _mark_queues; 100 volatile size_t _trashed_oops; 101 102 public: 103 explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : 104 WorkerTask("Purge SATB"), 105 _mark_queues(queues), 106 _trashed_oops(0) { 107 Threads::change_thread_claim_token(); 108 } 109 110 ~ShenandoahPurgeSATBTask() { 111 if (_trashed_oops > 0) { 112 log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers", _trashed_oops); 113 } 114 } 115 116 void work(uint worker_id) override { 117 ShenandoahParallelWorkerSession worker_session(worker_id); 118 ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set(); 119 ShenandoahFlushAllSATB flusher(satb_queues); 120 Threads::possibly_parallel_threads_do(true /* is_par */, &flusher); 121 122 ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id); 123 ShenandoahProcessOldSATB processor(mark_queue); 124 while (satb_queues.apply_closure_to_completed_buffer(&processor)) {} 125 126 Atomic::add(&_trashed_oops, processor.trashed_oops()); 127 } 128 }; 129 130 class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { 131 private: 132 uint _nworkers; 133 ShenandoahHeapRegion** _coalesce_and_fill_region_array; 134 uint _coalesce_and_fill_region_count; 135 volatile bool _is_preempted; 136 137 public: 138 ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, 139 ShenandoahHeapRegion** coalesce_and_fill_region_array, 140 uint region_count) : 141 WorkerTask("Shenandoah Concurrent Coalesce and Fill"), 142 _nworkers(nworkers), 143 _coalesce_and_fill_region_array(coalesce_and_fill_region_array), 144 _coalesce_and_fill_region_count(region_count), 145 _is_preempted(false) { 146 } 147 148 void work(uint worker_id) override { 149 ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_coalesce_and_fill, ShenandoahPhaseTimings::ScanClusters, worker_id); 150 for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { 151 ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; 152 if (r->is_humongous()) { 153 // There is only one object in this region and it is not garbage, 154 // so no need to coalesce or fill. 155 continue; 156 } 157 158 if (!r->oop_coalesce_and_fill(true)) { 159 // Coalesce and fill has been preempted 160 Atomic::store(&_is_preempted, true); 161 return; 162 } 163 } 164 } 165 166 // Value returned from is_completed() is only valid after all worker thread have terminated. 167 bool is_completed() { 168 return !Atomic::load(&_is_preempted); 169 } 170 }; 171 172 ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) 173 : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity), 174 _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)), 175 _old_heuristics(nullptr), 176 _region_balance(0), 177 _promoted_reserve(0), 178 _promoted_expended(0), 179 _promotion_potential(0), 180 _pad_for_promote_in_place(0), 181 _promotable_humongous_regions(0), 182 _promotable_regular_regions(0), 183 _is_parsable(true), 184 _card_scan(nullptr), 185 _state(WAITING_FOR_BOOTSTRAP), 186 _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION), 187 _min_growth_before_compaction ((ShenandoahMinOldGenGrowthPercent * FRACTIONAL_DENOMINATOR) / 100) 188 { 189 _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR; 190 // Always clear references for old generation 191 ref_processor()->set_soft_reference_policy(true); 192 193 if (ShenandoahCardBarrier) { 194 ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table(); 195 size_t card_count = card_table->cards_required(ShenandoahHeap::heap()->reserved_region().word_size()); 196 auto rs = new ShenandoahDirectCardMarkRememberedSet(card_table, card_count); 197 _card_scan = new ShenandoahScanRemembered(rs); 198 } 199 } 200 201 void ShenandoahOldGeneration::set_promoted_reserve(size_t new_val) { 202 shenandoah_assert_heaplocked_or_safepoint(); 203 _promoted_reserve = new_val; 204 } 205 206 size_t ShenandoahOldGeneration::get_promoted_reserve() const { 207 return _promoted_reserve; 208 } 209 210 void ShenandoahOldGeneration::augment_promoted_reserve(size_t increment) { 211 shenandoah_assert_heaplocked_or_safepoint(); 212 _promoted_reserve += increment; 213 } 214 215 void ShenandoahOldGeneration::reset_promoted_expended() { 216 shenandoah_assert_heaplocked_or_safepoint(); 217 Atomic::store(&_promoted_expended, (size_t) 0); 218 } 219 220 size_t ShenandoahOldGeneration::expend_promoted(size_t increment) { 221 shenandoah_assert_heaplocked_or_safepoint(); 222 assert(get_promoted_expended() + increment <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); 223 return Atomic::add(&_promoted_expended, increment); 224 } 225 226 size_t ShenandoahOldGeneration::unexpend_promoted(size_t decrement) { 227 return Atomic::sub(&_promoted_expended, decrement); 228 } 229 230 size_t ShenandoahOldGeneration::get_promoted_expended() const { 231 return Atomic::load(&_promoted_expended); 232 } 233 234 bool ShenandoahOldGeneration::can_allocate(const ShenandoahAllocRequest &req) const { 235 assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); 236 237 const size_t requested_bytes = req.size() * HeapWordSize; 238 // The promotion reserve may also be used for evacuations. If we can promote this object, 239 // then we can also evacuate it. 240 if (can_promote(requested_bytes)) { 241 // The promotion reserve should be able to accommodate this request. The request 242 // might still fail if alignment with the card table increases the size. The request 243 // may also fail if the heap is badly fragmented and the free set cannot find room for it. 244 return true; 245 } 246 247 if (req.type() == ShenandoahAllocRequest::_alloc_plab) { 248 // The promotion reserve cannot accommodate this plab request. Check if we still have room for 249 // evacuations. Note that we cannot really know how much of the plab will be used for evacuations, 250 // so here we only check that some evacuation reserve still exists. 251 return get_evacuation_reserve() > 0; 252 } 253 254 // This is a shared allocation request. We've already checked that it can't be promoted, so if 255 // it is a promotion, we return false. Otherwise, it is a shared evacuation request, and we allow 256 // the allocation to proceed. 257 return !req.is_promotion(); 258 } 259 260 void 261 ShenandoahOldGeneration::configure_plab_for_current_thread(const ShenandoahAllocRequest &req) { 262 // Note: Even when a mutator is performing a promotion outside a LAB, we use a 'shared_gc' request. 263 if (req.is_gc_alloc()) { 264 const size_t actual_size = req.actual_size() * HeapWordSize; 265 if (req.type() == ShenandoahAllocRequest::_alloc_plab) { 266 // We've created a new plab. Now we configure it whether it will be used for promotions 267 // and evacuations - or just evacuations. 268 Thread* thread = Thread::current(); 269 ShenandoahThreadLocalData::reset_plab_promoted(thread); 270 271 // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries). 272 // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread. 273 if (can_promote(actual_size)) { 274 // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. 275 // When we retire this plab, we'll unexpend what we don't really use. 276 expend_promoted(actual_size); 277 ShenandoahThreadLocalData::enable_plab_promotions(thread); 278 ShenandoahThreadLocalData::set_plab_actual_size(thread, actual_size); 279 } else { 280 // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. 281 ShenandoahThreadLocalData::disable_plab_promotions(thread); 282 ShenandoahThreadLocalData::set_plab_actual_size(thread, 0); 283 } 284 } else if (req.is_promotion()) { 285 // Shared promotion. 286 expend_promoted(actual_size); 287 } 288 } 289 } 290 291 size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const { 292 return _live_bytes_after_last_mark; 293 } 294 295 void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) { 296 if (bytes == 0) { 297 // Restart search for best old-gen size to the initial state 298 _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR; 299 _growth_before_compaction = INITIAL_GROWTH_BEFORE_COMPACTION; 300 } else { 301 _live_bytes_after_last_mark = bytes; 302 _growth_before_compaction /= 2; 303 if (_growth_before_compaction < _min_growth_before_compaction) { 304 _growth_before_compaction = _min_growth_before_compaction; 305 } 306 } 307 } 308 309 void ShenandoahOldGeneration::handle_failed_transfer() { 310 _old_heuristics->trigger_cannot_expand(); 311 } 312 313 size_t ShenandoahOldGeneration::usage_trigger_threshold() const { 314 size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR; 315 return result; 316 } 317 318 bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { 319 return region->is_old(); 320 } 321 322 void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { 323 ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl); 324 ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions_cl); 325 } 326 327 void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { 328 ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl); 329 ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl); 330 } 331 332 void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) { 333 ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress); 334 } 335 336 bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { 337 return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(); 338 } 339 340 void ShenandoahOldGeneration::cancel_marking() { 341 if (is_concurrent_mark_in_progress()) { 342 log_info(gc)("Abandon SATB buffers"); 343 ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); 344 } 345 346 ShenandoahGeneration::cancel_marking(); 347 } 348 349 void ShenandoahOldGeneration::cancel_gc() { 350 shenandoah_assert_safepoint(); 351 if (is_idle()) { 352 #ifdef ASSERT 353 validate_waiting_for_bootstrap(); 354 #endif 355 } else { 356 log_info(gc)("Terminating old gc cycle."); 357 // Stop marking 358 cancel_marking(); 359 // Stop tracking old regions 360 abandon_collection_candidates(); 361 // Remove old generation access to young generation mark queues 362 ShenandoahHeap::heap()->young_generation()->set_old_gen_task_queues(nullptr); 363 // Transition to IDLE now. 364 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP); 365 } 366 } 367 368 void ShenandoahOldGeneration::prepare_gc() { 369 // Now that we have made the old generation parsable, it is safe to reset the mark bitmap. 370 assert(state() != FILLING, "Cannot reset old without making it parsable"); 371 372 ShenandoahGeneration::prepare_gc(); 373 } 374 375 bool ShenandoahOldGeneration::entry_coalesce_and_fill() { 376 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 377 378 static const char* msg = "Coalescing and filling (OLD)"; 379 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill); 380 381 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 382 EventMark em("%s", msg); 383 ShenandoahWorkerScope scope(heap->workers(), 384 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 385 msg); 386 387 return coalesce_and_fill(); 388 } 389 390 // Make the old generation regions parsable, so they can be safely 391 // scanned when looking for objects in memory indicated by dirty cards. 392 bool ShenandoahOldGeneration::coalesce_and_fill() { 393 transition_to(FILLING); 394 395 // This code will see the same set of regions to fill on each resumption as it did 396 // on the initial run. That's okay because each region keeps track of its own coalesce 397 // and fill state. Regions that were filled on a prior attempt will not try to fill again. 398 uint coalesce_and_fill_regions_count = _old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); 399 assert(coalesce_and_fill_regions_count <= ShenandoahHeap::heap()->num_regions(), "Sanity"); 400 if (coalesce_and_fill_regions_count == 0) { 401 // No regions need to be filled. 402 abandon_collection_candidates(); 403 return true; 404 } 405 406 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 407 WorkerThreads* workers = heap->workers(); 408 uint nworkers = workers->active_workers(); 409 ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); 410 411 log_info(gc)("Starting (or resuming) coalesce-and-fill of " UINT32_FORMAT " old heap regions", coalesce_and_fill_regions_count); 412 workers->run_task(&task); 413 if (task.is_completed()) { 414 // We no longer need to track regions that need to be coalesced and filled. 415 abandon_collection_candidates(); 416 return true; 417 } else { 418 // Coalesce-and-fill has been preempted. We'll finish that effort in the future. Do not invoke 419 // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap 420 // and invokes set_mark_incomplete(). Coalesce-and-fill depends on the mark bitmap. 421 log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); 422 return false; 423 } 424 } 425 426 void ShenandoahOldGeneration::transfer_pointers_from_satb() { 427 ShenandoahHeap* heap = ShenandoahHeap::heap(); 428 shenandoah_assert_safepoint(); 429 assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); 430 log_info(gc)("Transfer SATB buffers"); 431 uint nworkers = heap->workers()->active_workers(); 432 StrongRootsScope scope(nworkers); 433 434 ShenandoahPurgeSATBTask purge_satb_task(task_queues()); 435 heap->workers()->run_task(&purge_satb_task); 436 } 437 438 bool ShenandoahOldGeneration::contains(oop obj) const { 439 return ShenandoahHeap::heap()->is_in_old(obj); 440 } 441 442 void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) { 443 ShenandoahHeap* heap = ShenandoahHeap::heap(); 444 assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); 445 446 { 447 ShenandoahGCPhase phase(concurrent ? 448 ShenandoahPhaseTimings::final_update_region_states : 449 ShenandoahPhaseTimings::degen_gc_final_update_region_states); 450 ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); 451 452 parallel_heap_region_iterate(&cl); 453 heap->assert_pinned_region_status(); 454 } 455 456 { 457 // This doesn't actually choose a collection set, but prepares a list of 458 // regions as 'candidates' for inclusion in a mixed collection. 459 ShenandoahGCPhase phase(concurrent ? 460 ShenandoahPhaseTimings::choose_cset : 461 ShenandoahPhaseTimings::degen_gc_choose_cset); 462 ShenandoahHeapLocker locker(heap->lock()); 463 _old_heuristics->prepare_for_old_collections(); 464 } 465 466 { 467 // Though we did not choose a collection set above, we still may have 468 // freed up immediate garbage regions so proceed with rebuilding the free set. 469 ShenandoahGCPhase phase(concurrent ? 470 ShenandoahPhaseTimings::final_rebuild_freeset : 471 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); 472 ShenandoahHeapLocker locker(heap->lock()); 473 size_t cset_young_regions, cset_old_regions; 474 size_t first_old, last_old, num_old; 475 heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); 476 // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here 477 // is in case there was any immediate old garbage identified. 478 heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old); 479 } 480 } 481 482 const char* ShenandoahOldGeneration::state_name(State state) { 483 switch (state) { 484 case WAITING_FOR_BOOTSTRAP: return "Waiting for Bootstrap"; 485 case FILLING: return "Coalescing"; 486 case BOOTSTRAPPING: return "Bootstrapping"; 487 case MARKING: return "Marking"; 488 case EVACUATING: return "Evacuating"; 489 case EVACUATING_AFTER_GLOBAL: return "Evacuating (G)"; 490 default: 491 ShouldNotReachHere(); 492 return "Unknown"; 493 } 494 } 495 496 void ShenandoahOldGeneration::transition_to(State new_state) { 497 if (_state != new_state) { 498 log_info(gc)("Old generation transition from %s to %s", state_name(_state), state_name(new_state)); 499 validate_transition(new_state); 500 _state = new_state; 501 } 502 } 503 504 #ifdef ASSERT 505 // This diagram depicts the expected state transitions for marking the old generation 506 // and preparing for old collections. When a young generation cycle executes, the 507 // remembered set scan must visit objects in old regions. Visiting an object which 508 // has become dead on previous old cycles will result in crashes. To avoid visiting 509 // such objects, the remembered set scan will use the old generation mark bitmap when 510 // possible. It is _not_ possible to use the old generation bitmap when old marking 511 // is active (bitmap is not complete). For this reason, the old regions are made 512 // parsable _before_ the old generation bitmap is reset. The diagram does not depict 513 // cancellation of old collections by global or full collections. 514 // 515 // When a global collection supersedes an old collection, the global mark still 516 // "completes" the old mark bitmap. Subsequent remembered set scans may use the 517 // old generation mark bitmap, but any uncollected old regions must still be made parsable 518 // before the next old generation cycle begins. For this reason, a global collection may 519 // create mixed collection candidates and coalesce and fill candidates and will put 520 // the old generation in the respective states (EVACUATING or FILLING). After a Full GC, 521 // the mark bitmaps are all reset, all regions are parsable and the mark context will 522 // not be "complete". After a Full GC, remembered set scans will _not_ use the mark bitmap 523 // and we expect the old generation to be waiting for bootstrap. 524 // 525 // +-----------------+ 526 // +------------> | FILLING | <---+ 527 // | +--------> | | | 528 // | | +-----------------+ | 529 // | | | | 530 // | | | Filling Complete | <-> A global collection may 531 // | | v | move the old generation 532 // | | +-----------------+ | directly from waiting for 533 // +-- |-- |--------> | WAITING | | bootstrap to filling or 534 // | | | +---- | FOR BOOTSTRAP | ----+ evacuating. It may also 535 // | | | | +-----------------+ move from filling to waiting 536 // | | | | | for bootstrap. 537 // | | | | | Reset Bitmap 538 // | | | | v 539 // | | | | +-----------------+ +----------------------+ 540 // | | | | | BOOTSTRAP | <-> | YOUNG GC | 541 // | | | | | | | (RSet Parses Region) | 542 // | | | | +-----------------+ +----------------------+ 543 // | | | | | 544 // | | | | | Old Marking 545 // | | | | v 546 // | | | | +-----------------+ +----------------------+ 547 // | | | | | MARKING | <-> | YOUNG GC | 548 // | | +--------- | | | (RSet Parses Region) | 549 // | | | +-----------------+ +----------------------+ 550 // | | | | 551 // | | | | Has Evacuation Candidates 552 // | | | v 553 // | | | +-----------------+ +--------------------+ 554 // | | +---> | EVACUATING | <-> | YOUNG GC | 555 // | +------------- | | | (RSet Uses Bitmap) | 556 // | +-----------------+ +--------------------+ 557 // | | 558 // | | Global Cycle Coalesces and Fills Old Regions 559 // | v 560 // | +-----------------+ +--------------------+ 561 // +----------------- | EVACUATING | <-> | YOUNG GC | 562 // | AFTER GLOBAL | | (RSet Uses Bitmap) | 563 // +-----------------+ +--------------------+ 564 // 565 // 566 void ShenandoahOldGeneration::validate_transition(State new_state) { 567 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 568 switch (new_state) { 569 case FILLING: 570 assert(_state != BOOTSTRAPPING, "Cannot begin making old regions parsable after bootstrapping"); 571 assert(is_mark_complete(), "Cannot begin filling without first completing marking, state is '%s'", state_name(_state)); 572 assert(_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot begin filling without something to fill."); 573 break; 574 case WAITING_FOR_BOOTSTRAP: 575 // GC cancellation can send us back here from any state. 576 validate_waiting_for_bootstrap(); 577 break; 578 case BOOTSTRAPPING: 579 assert(_state == WAITING_FOR_BOOTSTRAP, "Cannot reset bitmap without making old regions parsable, state is '%s'", state_name(_state)); 580 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates"); 581 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parsable."); 582 break; 583 case MARKING: 584 assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state)); 585 assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); 586 assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); 587 break; 588 case EVACUATING_AFTER_GLOBAL: 589 assert(_state == EVACUATING, "Must have been evacuating, state is '%s'", state_name(_state)); 590 break; 591 case EVACUATING: 592 assert(_state == WAITING_FOR_BOOTSTRAP || _state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state)); 593 assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); 594 break; 595 default: 596 fatal("Unknown new state"); 597 } 598 } 599 600 bool ShenandoahOldGeneration::validate_waiting_for_bootstrap() { 601 ShenandoahHeap* heap = ShenandoahHeap::heap(); 602 assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become ready for bootstrap during old mark."); 603 assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become ready for bootstrap when still setup for bootstrapping."); 604 assert(!is_concurrent_mark_in_progress(), "Cannot be marking in IDLE"); 605 assert(!heap->young_generation()->is_bootstrap_cycle(), "Cannot have old mark queues if IDLE"); 606 assert(!_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot have coalesce and fill candidates in IDLE"); 607 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot have mixed collection candidates in IDLE"); 608 return true; 609 } 610 #endif 611 612 ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { 613 _old_heuristics = new ShenandoahOldHeuristics(this, ShenandoahGenerationalHeap::heap()); 614 _old_heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval); 615 _heuristics = _old_heuristics; 616 return _heuristics; 617 } 618 619 void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) { 620 heuristics()->record_success_concurrent(); 621 ShenandoahHeap::heap()->shenandoah_policy()->record_success_old(); 622 } 623 624 void ShenandoahOldGeneration::handle_failed_evacuation() { 625 if (_failed_evacuation.try_set()) { 626 log_info(gc)("Old gen evac failure."); 627 } 628 } 629 630 void ShenandoahOldGeneration::handle_failed_promotion(Thread* thread, size_t size) { 631 // We squelch excessive reports to reduce noise in logs. 632 const size_t MaxReportsPerEpoch = 4; 633 static size_t last_report_epoch = 0; 634 static size_t epoch_report_count = 0; 635 auto heap = ShenandoahGenerationalHeap::heap(); 636 637 size_t promotion_reserve; 638 size_t promotion_expended; 639 640 const size_t gc_id = heap->control_thread()->get_gc_id(); 641 642 if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) { 643 { 644 // Promotion failures should be very rare. Invest in providing useful diagnostic info. 645 ShenandoahHeapLocker locker(heap->lock()); 646 promotion_reserve = get_promoted_reserve(); 647 promotion_expended = get_promoted_expended(); 648 } 649 PLAB* const plab = ShenandoahThreadLocalData::plab(thread); 650 const size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining(); 651 const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled"; 652 653 log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT 654 ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT 655 ", old capacity: " SIZE_FORMAT ", old_used: " SIZE_FORMAT ", old unaffiliated regions: " SIZE_FORMAT, 656 size * HeapWordSize, plab == nullptr? "no": "yes", 657 words_remaining * HeapWordSize, promote_enabled, promotion_reserve, promotion_expended, 658 max_capacity(), used(), free_unaffiliated_regions()); 659 660 if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { 661 log_info(gc, ergo)("Squelching additional promotion failure reports for current epoch"); 662 } else if (gc_id != last_report_epoch) { 663 last_report_epoch = gc_id; 664 epoch_report_count = 1; 665 } 666 } 667 } 668 669 void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, bool promotion) { 670 // Only register the copy of the object that won the evacuation race. 671 _card_scan->register_object_without_lock(obj); 672 673 // Mark the entire range of the evacuated object as dirty. At next remembered set scan, 674 // we will clear dirty bits that do not hold interesting pointers. It's more efficient to 675 // do this in batch, in a background GC thread than to try to carefully dirty only cards 676 // that hold interesting pointers right now. 677 _card_scan->mark_range_as_dirty(obj, words); 678 679 if (promotion) { 680 // This evacuation was a promotion, track this as allocation against old gen 681 increase_allocated(words * HeapWordSize); 682 } 683 } 684 685 bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() { 686 return _old_heuristics->unprocessed_old_collection_candidates() > 0; 687 } 688 689 size_t ShenandoahOldGeneration::unprocessed_collection_candidates_live_memory() { 690 return _old_heuristics->unprocessed_old_collection_candidates_live_memory(); 691 } 692 693 void ShenandoahOldGeneration::abandon_collection_candidates() { 694 _old_heuristics->abandon_collection_candidates(); 695 } 696 697 void ShenandoahOldGeneration::prepare_for_mixed_collections_after_global_gc() { 698 assert(is_mark_complete(), "Expected old generation mark to be complete after global cycle."); 699 _old_heuristics->prepare_for_old_collections(); 700 log_info(gc)("After choosing global collection set, mixed candidates: " UINT32_FORMAT ", coalescing candidates: " SIZE_FORMAT, 701 _old_heuristics->unprocessed_old_collection_candidates(), 702 _old_heuristics->coalesce_and_fill_candidates_count()); 703 } 704 705 void ShenandoahOldGeneration::parallel_region_iterate_free(ShenandoahHeapRegionClosure* cl) { 706 // Iterate over old and free regions (exclude young). 707 ShenandoahExcludeRegionClosure<YOUNG_GENERATION> exclude_cl(cl); 708 ShenandoahGeneration::parallel_region_iterate_free(&exclude_cl); 709 } 710 711 void ShenandoahOldGeneration::set_parsable(bool parsable) { 712 _is_parsable = parsable; 713 if (_is_parsable) { 714 // The current state would have been chosen during final mark of the global 715 // collection, _before_ any decisions about class unloading have been made. 716 // 717 // After unloading classes, we have made the old generation regions parsable. 718 // We can skip filling or transition to a state that knows everything has 719 // already been filled. 720 switch (state()) { 721 case ShenandoahOldGeneration::EVACUATING: 722 transition_to(ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL); 723 break; 724 case ShenandoahOldGeneration::FILLING: 725 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Expected no mixed collection candidates"); 726 assert(_old_heuristics->coalesce_and_fill_candidates_count() > 0, "Expected coalesce and fill candidates"); 727 // When the heuristic put the old generation in this state, it didn't know 728 // that we would unload classes and make everything parsable. But, we know 729 // that now so we can override this state. 730 abandon_collection_candidates(); 731 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP); 732 break; 733 default: 734 // We can get here during a full GC. The full GC will cancel anything 735 // happening in the old generation and return it to the waiting for bootstrap 736 // state. The full GC will then record that the old regions are parsable 737 // after rebuilding the remembered set. 738 assert(is_idle(), "Unexpected state %s at end of global GC", state_name()); 739 break; 740 } 741 } 742 } 743 744 void ShenandoahOldGeneration::complete_mixed_evacuations() { 745 assert(is_doing_mixed_evacuations(), "Mixed evacuations should be in progress"); 746 if (!_old_heuristics->has_coalesce_and_fill_candidates()) { 747 // No candidate regions to coalesce and fill 748 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP); 749 return; 750 } 751 752 if (state() == ShenandoahOldGeneration::EVACUATING) { 753 transition_to(ShenandoahOldGeneration::FILLING); 754 return; 755 } 756 757 // Here, we have no more candidates for mixed collections. The candidates for coalescing 758 // and filling have already been processed during the global cycle, so there is nothing 759 // more to do. 760 assert(state() == ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL, "Should be evacuating after a global cycle"); 761 abandon_collection_candidates(); 762 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP); 763 } 764 765 void ShenandoahOldGeneration::abandon_mixed_evacuations() { 766 switch(state()) { 767 case ShenandoahOldGeneration::EVACUATING: 768 transition_to(ShenandoahOldGeneration::FILLING); 769 break; 770 case ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL: 771 abandon_collection_candidates(); 772 transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP); 773 break; 774 default: 775 ShouldNotReachHere(); 776 break; 777 } 778 } 779 780 void ShenandoahOldGeneration::clear_cards_for(ShenandoahHeapRegion* region) { 781 _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom())); 782 } 783 784 void ShenandoahOldGeneration::mark_card_as_dirty(void* location) { 785 _card_scan->mark_card_as_dirty((HeapWord*)location); 786 }