1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 26 #include "precompiled.hpp" 27 28 #include "gc/shared/strongRootsScope.hpp" 29 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 30 #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" 31 #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" 32 #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" 33 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" 34 #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" 35 #include "gc/shenandoah/shenandoahAsserts.hpp" 36 #include "gc/shenandoah/shenandoahFreeSet.hpp" 37 #include "gc/shenandoah/shenandoahHeap.hpp" 38 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 39 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 40 #include "gc/shenandoah/shenandoahMarkClosures.hpp" 41 #include "gc/shenandoah/shenandoahMark.inline.hpp" 42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 43 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 44 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 45 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 46 #include "gc/shenandoah/shenandoahStringDedup.hpp" 47 #include "gc/shenandoah/shenandoahUtils.hpp" 48 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 49 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 50 #include "prims/jvmtiTagMap.hpp" 51 #include "runtime/threads.hpp" 52 #include "utilities/events.hpp" 53 54 class ShenandoahFlushAllSATB : public ThreadClosure { 55 private: 56 SATBMarkQueueSet& _satb_qset; 57 58 public: 59 explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : 60 _satb_qset(satb_qset) {} 61 62 void do_thread(Thread* thread) { 63 // Transfer any partial buffer to the qset for completed buffer processing. 64 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); 65 } 66 }; 67 68 class ShenandoahProcessOldSATB : public SATBBufferClosure { 69 private: 70 ShenandoahObjToScanQueue* _queue; 71 ShenandoahHeap* _heap; 72 ShenandoahMarkingContext* const _mark_context; 73 size_t _trashed_oops; 74 75 public: 76 explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) : 77 _queue(q), 78 _heap(ShenandoahHeap::heap()), 79 _mark_context(_heap->marking_context()), 80 _trashed_oops(0) {} 81 82 void do_buffer(void** buffer, size_t size) { 83 assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); 84 for (size_t i = 0; i < size; ++i) { 85 oop *p = (oop *) &buffer[i]; 86 ShenandoahHeapRegion* region = _heap->heap_region_containing(*p); 87 if (region->is_old() && region->is_active()) { 88 ShenandoahMark::mark_through_ref<oop, OLD>(p, _queue, nullptr, _mark_context, false); 89 } else { 90 _trashed_oops++; 91 } 92 } 93 } 94 95 size_t trashed_oops() { 96 return _trashed_oops; 97 } 98 }; 99 100 class ShenandoahPurgeSATBTask : public WorkerTask { 101 private: 102 ShenandoahObjToScanQueueSet* _mark_queues; 103 volatile size_t _trashed_oops; 104 105 public: 106 explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : 107 WorkerTask("Purge SATB"), 108 _mark_queues(queues), 109 _trashed_oops(0) { 110 Threads::change_thread_claim_token(); 111 } 112 113 ~ShenandoahPurgeSATBTask() { 114 if (_trashed_oops > 0) { 115 log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers", _trashed_oops); 116 } 117 } 118 119 void work(uint worker_id) { 120 ShenandoahParallelWorkerSession worker_session(worker_id); 121 ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set(); 122 ShenandoahFlushAllSATB flusher(satb_queues); 123 Threads::possibly_parallel_threads_do(true /* is_par */, &flusher); 124 125 ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id); 126 ShenandoahProcessOldSATB processor(mark_queue); 127 while (satb_queues.apply_closure_to_completed_buffer(&processor)) {} 128 129 Atomic::add(&_trashed_oops, processor.trashed_oops()); 130 } 131 }; 132 133 class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { 134 private: 135 uint _nworkers; 136 ShenandoahHeapRegion** _coalesce_and_fill_region_array; 137 uint _coalesce_and_fill_region_count; 138 volatile bool _is_preempted; 139 140 public: 141 ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, 142 ShenandoahHeapRegion** coalesce_and_fill_region_array, 143 uint region_count) : 144 WorkerTask("Shenandoah Concurrent Coalesce and Fill"), 145 _nworkers(nworkers), 146 _coalesce_and_fill_region_array(coalesce_and_fill_region_array), 147 _coalesce_and_fill_region_count(region_count), 148 _is_preempted(false) { 149 } 150 151 void work(uint worker_id) { 152 for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { 153 ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; 154 if (r->is_humongous()) { 155 // There is only one object in this region and it is not garbage, 156 // so no need to coalesce or fill. 157 continue; 158 } 159 160 if (!r->oop_fill_and_coalesce()) { 161 // Coalesce and fill has been preempted 162 Atomic::store(&_is_preempted, true); 163 return; 164 } 165 } 166 } 167 168 // Value returned from is_completed() is only valid after all worker thread have terminated. 169 bool is_completed() { 170 return !Atomic::load(&_is_preempted); 171 } 172 }; 173 174 ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) 175 : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity), 176 _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)), 177 _state(IDLE), 178 _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION) 179 { 180 _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR; 181 // Always clear references for old generation 182 ref_processor()->set_soft_reference_policy(true); 183 } 184 185 size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const { 186 return _live_bytes_after_last_mark; 187 } 188 189 void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) { 190 _live_bytes_after_last_mark = bytes; 191 if (_growth_before_compaction > MINIMUM_GROWTH_BEFORE_COMPACTION) { 192 _growth_before_compaction /= 2; 193 } 194 } 195 196 size_t ShenandoahOldGeneration::usage_trigger_threshold() const { 197 size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR; 198 return result; 199 } 200 201 bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { 202 // TODO: Should this be region->is_old() instead? 203 return !region->is_young(); 204 } 205 206 void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { 207 ShenandoahGenerationRegionClosure<OLD> old_regions(cl); 208 ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions); 209 } 210 211 void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { 212 ShenandoahGenerationRegionClosure<OLD> old_regions(cl); 213 ShenandoahHeap::heap()->heap_region_iterate(&old_regions); 214 } 215 216 void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) { 217 ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress); 218 } 219 220 bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { 221 return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(); 222 } 223 224 void ShenandoahOldGeneration::cancel_marking() { 225 if (is_concurrent_mark_in_progress()) { 226 log_info(gc)("Abandon SATB buffers"); 227 ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); 228 } 229 230 ShenandoahGeneration::cancel_marking(); 231 } 232 233 void ShenandoahOldGeneration::prepare_gc() { 234 // Make the old generation regions parseable, so they can be safely 235 // scanned when looking for objects in memory indicated by dirty cards. 236 if (entry_coalesce_and_fill()) { 237 // Now that we have made the old generation parseable, it is safe to reset the mark bitmap. 238 static const char* msg = "Concurrent reset (OLD)"; 239 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old); 240 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 241 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 242 msg); 243 ShenandoahGeneration::prepare_gc(); 244 } 245 // Else, coalesce-and-fill has been preempted and we'll finish that effort in the future. Do not invoke 246 // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap 247 // and invokes set_mark_incomplete(). Coalesce-and-fill depends on the mark bitmap. 248 } 249 250 bool ShenandoahOldGeneration::entry_coalesce_and_fill() { 251 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 252 253 static const char* msg = "Coalescing and filling (OLD)"; 254 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); 255 256 // TODO: I don't think we're using these concurrent collection counters correctly. 257 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 258 EventMark em("%s", msg); 259 ShenandoahWorkerScope scope(heap->workers(), 260 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 261 msg); 262 263 return coalesce_and_fill(); 264 } 265 266 bool ShenandoahOldGeneration::coalesce_and_fill() { 267 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 268 heap->set_prepare_for_old_mark_in_progress(true); 269 transition_to(FILLING); 270 271 ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); 272 WorkerThreads* workers = heap->workers(); 273 uint nworkers = workers->active_workers(); 274 275 log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); 276 277 // This code will see the same set of regions to fill on each resumption as it did 278 // on the initial run. That's okay because each region keeps track of its own coalesce 279 // and fill state. Regions that were filled on a prior attempt will not try to fill again. 280 uint coalesce_and_fill_regions_count = old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); 281 assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); 282 ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); 283 284 workers->run_task(&task); 285 if (task.is_completed()) { 286 // Remember that we're done with coalesce-and-fill. 287 heap->set_prepare_for_old_mark_in_progress(false); 288 old_heuristics->abandon_collection_candidates(); 289 return true; 290 } else { 291 // Otherwise, we were preempted before the work was done. 292 log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); 293 return false; 294 } 295 } 296 297 void ShenandoahOldGeneration::transfer_pointers_from_satb() { 298 ShenandoahHeap* heap = ShenandoahHeap::heap(); 299 shenandoah_assert_safepoint(); 300 assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); 301 log_info(gc)("Transfer SATB buffers"); 302 uint nworkers = heap->workers()->active_workers(); 303 StrongRootsScope scope(nworkers); 304 305 ShenandoahPurgeSATBTask purge_satb_task(task_queues()); 306 heap->workers()->run_task(&purge_satb_task); 307 } 308 309 bool ShenandoahOldGeneration::contains(oop obj) const { 310 return ShenandoahHeap::heap()->is_in_old(obj); 311 } 312 313 void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) { 314 ShenandoahHeap* heap = ShenandoahHeap::heap(); 315 assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); 316 317 { 318 ShenandoahGCPhase phase(concurrent ? 319 ShenandoahPhaseTimings::final_update_region_states : 320 ShenandoahPhaseTimings::degen_gc_final_update_region_states); 321 ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); 322 323 parallel_heap_region_iterate(&cl); 324 heap->assert_pinned_region_status(); 325 } 326 327 { 328 // This doesn't actually choose a collection set, but prepares a list of 329 // regions as 'candidates' for inclusion in a mixed collection. 330 ShenandoahGCPhase phase(concurrent ? 331 ShenandoahPhaseTimings::choose_cset : 332 ShenandoahPhaseTimings::degen_gc_choose_cset); 333 ShenandoahHeapLocker locker(heap->lock()); 334 _old_heuristics->prepare_for_old_collections(); 335 } 336 337 { 338 // Though we did not choose a collection set above, we still may have 339 // freed up immediate garbage regions so proceed with rebuilding the free set. 340 ShenandoahGCPhase phase(concurrent ? 341 ShenandoahPhaseTimings::final_rebuild_freeset : 342 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); 343 ShenandoahHeapLocker locker(heap->lock()); 344 size_t cset_young_regions, cset_old_regions; 345 heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions); 346 // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here 347 // is in case there was any immediate old garbage identified. 348 heap->free_set()->rebuild(cset_young_regions, cset_old_regions); 349 } 350 } 351 352 const char* ShenandoahOldGeneration::state_name(State state) { 353 switch (state) { 354 case IDLE: return "Idle"; 355 case FILLING: return "Coalescing"; 356 case BOOTSTRAPPING: return "Bootstrapping"; 357 case MARKING: return "Marking"; 358 case WAITING_FOR_EVAC: return "Waiting for evacuation"; 359 case WAITING_FOR_FILL: return "Waiting for fill"; 360 default: 361 ShouldNotReachHere(); 362 return "Unknown"; 363 } 364 } 365 366 void ShenandoahOldGeneration::transition_to(State new_state) { 367 if (_state != new_state) { 368 log_info(gc)("Old generation transition from %s to %s", state_name(_state), state_name(new_state)); 369 validate_transition(new_state); 370 _state = new_state; 371 } 372 } 373 374 #ifdef ASSERT 375 // This diagram depicts the expected state transitions for marking the old generation 376 // and preparing for old collections. When a young generation cycle executes, the 377 // remembered set scan must visit objects in old regions. Visiting an object which 378 // has become dead on previous old cycles will result in crashes. To avoid visiting 379 // such objects, the remembered set scan will use the old generation mark bitmap when 380 // possible. It is _not_ possible to use the old generation bitmap when old marking 381 // is active (bitmap is not complete). For this reason, the old regions are made 382 // parseable _before_ the old generation bitmap is reset. The diagram does not depict 383 // cancellation of old collections by global or full collections. However, it does 384 // depict a transition from IDLE to WAITING_FOR_FILL, which is allowed after a global 385 // cycle ends. Also note that a global collection will cause any evacuation or fill 386 // candidates to be abandoned, returning the old generation to the idle state. 387 // 388 // +----------------> +-----------------+ 389 // | +------------> | IDLE | 390 // | | +--------> | | 391 // | | | +-----------------+ 392 // | | | | 393 // | | | | Begin Old Mark 394 // | | | v 395 // | | | +-----------------+ +--------------------+ 396 // | | | | FILLING | <-> | YOUNG GC | 397 // | | | +---> | | | (RSet Uses Bitmap) | 398 // | | | | +-----------------+ +--------------------+ 399 // | | | | | 400 // | | | | | Reset Bitmap 401 // | | | | v 402 // | | | | +-----------------+ 403 // | | | | | BOOTSTRAP | 404 // | | | | | | 405 // | | | | +-----------------+ 406 // | | | | | 407 // | | | | | Continue Marking 408 // | | | | v 409 // | | | | +-----------------+ +----------------------+ 410 // | | | | | MARKING | <-> | YOUNG GC | 411 // | | +----|-----| | | (RSet Parses Region) | 412 // | | | +-----------------+ +----------------------+ 413 // | | | | 414 // | | | | Has Candidates 415 // | | | v 416 // | | | +-----------------+ 417 // | | | | WAITING FOR | 418 // | +--------|---> | EVACUATIONS | 419 // | | +-----------------+ 420 // | | | 421 // | | | All Candidates are Pinned 422 // | | v 423 // | | +-----------------+ 424 // | +---- | WAITING FOR | 425 // +----------------> | FILLING | 426 // +-----------------+ 427 // 428 void ShenandoahOldGeneration::validate_transition(State new_state) { 429 ShenandoahHeap* heap = ShenandoahHeap::heap(); 430 switch (new_state) { 431 case IDLE: 432 // GC cancellation can send us back to IDLE from any state. 433 assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become idle during old mark."); 434 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); 435 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); 436 assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); 437 break; 438 case FILLING: 439 assert(_state == IDLE || _state == WAITING_FOR_FILL, "Cannot begin filling without first completing evacuations, state is '%s'", state_name(_state)); 440 assert(heap->is_prepare_for_old_mark_in_progress(), "Should be preparing for old mark now."); 441 break; 442 case BOOTSTRAPPING: 443 assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable, state is '%s'", state_name(_state)); 444 assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates"); 445 assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parseable."); 446 break; 447 case MARKING: 448 assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state)); 449 assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); 450 assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); 451 break; 452 case WAITING_FOR_EVAC: 453 assert(_state == IDLE || _state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state)); 454 assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); 455 break; 456 case WAITING_FOR_FILL: 457 assert(_state == IDLE || _state == MARKING || _state == WAITING_FOR_EVAC, "Cannot begin filling without first marking or evacuating, state is '%s'", state_name(_state)); 458 assert(_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot wait for fill without something to fill."); 459 break; 460 default: 461 fatal("Unknown new state"); 462 } 463 } 464 #endif 465 466 ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { 467 _old_heuristics = new ShenandoahOldHeuristics(this); 468 _old_heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval); 469 _heuristics = _old_heuristics; 470 return _heuristics; 471 } 472 473 void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) { 474 heuristics()->record_success_concurrent(abbreviated); 475 ShenandoahHeap::heap()->shenandoah_policy()->record_success_old(); 476 }