1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 28 #include "gc/shenandoah/shenandoahFreeSet.hpp" 29 #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp" 30 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp" 31 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 32 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 33 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 34 #include "gc/shenandoah/shenandoahInitLogger.hpp" 35 #include "gc/shenandoah/shenandoahMemoryPool.hpp" 36 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 37 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 38 #include "gc/shenandoah/shenandoahRegulatorThread.hpp" 39 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 40 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 41 #include "gc/shenandoah/shenandoahUtils.hpp" 42 #include "logging/log.hpp" 43 44 45 class ShenandoahGenerationalInitLogger : public ShenandoahInitLogger { 46 public: 47 static void print() { 48 ShenandoahGenerationalInitLogger logger; 49 logger.print_all(); 50 } 51 52 void print_heap() override { 53 ShenandoahInitLogger::print_heap(); 54 55 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 56 57 ShenandoahYoungGeneration* young = heap->young_generation(); 58 log_info(gc, init)("Young Generation Soft Size: " EXACTFMT, EXACTFMTARGS(young->soft_max_capacity())); 59 log_info(gc, init)("Young Generation Max: " EXACTFMT, EXACTFMTARGS(young->max_capacity())); 60 61 ShenandoahOldGeneration* old = heap->old_generation(); 62 log_info(gc, init)("Old Generation Soft Size: " EXACTFMT, EXACTFMTARGS(old->soft_max_capacity())); 63 log_info(gc, init)("Old Generation Max: " EXACTFMT, EXACTFMTARGS(old->max_capacity())); 64 } 65 66 protected: 67 void print_gc_specific() override { 68 ShenandoahInitLogger::print_gc_specific(); 69 70 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 71 log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name()); 72 log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name()); 73 } 74 }; 75 76 ShenandoahGenerationalHeap* ShenandoahGenerationalHeap::heap() { 77 shenandoah_assert_generational(); 78 CollectedHeap* heap = Universe::heap(); 79 return checked_cast<ShenandoahGenerationalHeap*>(heap); 80 } 81 82 size_t ShenandoahGenerationalHeap::calculate_min_plab() { 83 return align_up(PLAB::min_size(), CardTable::card_size_in_words()); 84 } 85 86 size_t ShenandoahGenerationalHeap::calculate_max_plab() { 87 size_t MaxTLABSizeWords = ShenandoahHeapRegion::max_tlab_size_words(); 88 return ((ShenandoahMaxEvacLABRatio > 0)? 89 align_down(MIN2(MaxTLABSizeWords, PLAB::min_size() * ShenandoahMaxEvacLABRatio), CardTable::card_size_in_words()): 90 align_down(MaxTLABSizeWords, CardTable::card_size_in_words())); 91 } 92 93 // Returns size in bytes 94 size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const { 95 return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available()); 96 } 97 98 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) : 99 ShenandoahHeap(policy), 100 _min_plab_size(calculate_min_plab()), 101 _max_plab_size(calculate_max_plab()), 102 _regulator_thread(nullptr), 103 _young_gen_memory_pool(nullptr), 104 _old_gen_memory_pool(nullptr) { 105 assert(is_aligned(_min_plab_size, CardTable::card_size_in_words()), "min_plab_size must be aligned"); 106 assert(is_aligned(_max_plab_size, CardTable::card_size_in_words()), "max_plab_size must be aligned"); 107 } 108 109 void ShenandoahGenerationalHeap::print_init_logger() const { 110 ShenandoahGenerationalInitLogger logger; 111 logger.print_all(); 112 } 113 114 void ShenandoahGenerationalHeap::initialize_serviceability() { 115 assert(mode()->is_generational(), "Only for the generational mode"); 116 _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this); 117 _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this); 118 cycle_memory_manager()->add_pool(_young_gen_memory_pool); 119 cycle_memory_manager()->add_pool(_old_gen_memory_pool); 120 stw_memory_manager()->add_pool(_young_gen_memory_pool); 121 stw_memory_manager()->add_pool(_old_gen_memory_pool); 122 } 123 124 GrowableArray<MemoryPool*> ShenandoahGenerationalHeap::memory_pools() { 125 assert(mode()->is_generational(), "Only for the generational mode"); 126 GrowableArray<MemoryPool*> memory_pools(2); 127 memory_pools.append(_young_gen_memory_pool); 128 memory_pools.append(_old_gen_memory_pool); 129 return memory_pools; 130 } 131 132 void ShenandoahGenerationalHeap::initialize_controller() { 133 auto control_thread = new ShenandoahGenerationalControlThread(); 134 _control_thread = control_thread; 135 _regulator_thread = new ShenandoahRegulatorThread(control_thread); 136 } 137 138 void ShenandoahGenerationalHeap::gc_threads_do(ThreadClosure* tcl) const { 139 if (!shenandoah_policy()->is_at_shutdown()) { 140 ShenandoahHeap::gc_threads_do(tcl); 141 tcl->do_thread(regulator_thread()); 142 } 143 } 144 145 void ShenandoahGenerationalHeap::stop() { 146 regulator_thread()->stop(); 147 ShenandoahHeap::stop(); 148 } 149 150 oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) { 151 assert(thread == Thread::current(), "Expected thread parameter to be current thread."); 152 if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) { 153 // This thread went through the OOM during evac protocol and it is safe to return 154 // the forward pointer. It must not attempt to evacuate anymore. 155 return ShenandoahBarrierSet::resolve_forwarded(p); 156 } 157 158 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); 159 160 ShenandoahHeapRegion* r = heap_region_containing(p); 161 assert(!r->is_humongous(), "never evacuate humongous objects"); 162 163 ShenandoahAffiliation target_gen = r->affiliation(); 164 if (active_generation()->is_young() && target_gen == YOUNG_GENERATION) { 165 markWord mark = p->mark(); 166 if (mark.is_marked()) { 167 // Already forwarded. 168 return ShenandoahBarrierSet::resolve_forwarded(p); 169 } 170 171 if (mark.has_displaced_mark_helper()) { 172 // We don't want to deal with MT here just to ensure we read the right mark word. 173 // Skip the potential promotion attempt for this one. 174 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) { 175 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION); 176 if (result != nullptr) { 177 return result; 178 } 179 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen. 180 } 181 } 182 return try_evacuate_object(p, thread, r, target_gen); 183 } 184 185 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating 186 // to OLD_GENERATION. 187 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, 188 ShenandoahAffiliation target_gen) { 189 bool alloc_from_lab = true; 190 bool has_plab = false; 191 HeapWord* copy = nullptr; 192 size_t size = p->size(); 193 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young(); 194 195 #ifdef ASSERT 196 if (ShenandoahOOMDuringEvacALot && 197 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call 198 copy = nullptr; 199 } else { 200 #endif 201 if (UseTLAB) { 202 switch (target_gen) { 203 case YOUNG_GENERATION: { 204 copy = allocate_from_gclab(thread, size); 205 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) { 206 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting 207 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations. 208 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size()); 209 copy = allocate_from_gclab(thread, size); 210 // If we still get nullptr, we'll try a shared allocation below. 211 } 212 break; 213 } 214 case OLD_GENERATION: { 215 assert(mode()->is_generational(), "OLD Generation only exists in generational mode"); 216 PLAB* plab = ShenandoahThreadLocalData::plab(thread); 217 if (plab != nullptr) { 218 has_plab = true; 219 } 220 copy = allocate_from_plab(thread, size, is_promotion); 221 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) && 222 ShenandoahThreadLocalData::plab_retries_enabled(thread)) { 223 // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because 224 // the requested object does not fit within the current plab but the plab still has an "abundance" of memory, 225 // where abundance is defined as >= ShenGenHeap::plab_min_size(). In the former case, we try resetting the desired 226 // PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. 227 228 // In this situation, PLAB memory is precious. We'll try to preserve our existing PLAB by forcing 229 // this particular allocation to be shared. 230 if (plab->words_remaining() < plab_min_size()) { 231 ShenandoahThreadLocalData::set_plab_size(thread, plab_min_size()); 232 copy = allocate_from_plab(thread, size, is_promotion); 233 // If we still get nullptr, we'll try a shared allocation below. 234 if (copy == nullptr) { 235 // If retry fails, don't continue to retry until we have success (probably in next GC pass) 236 ShenandoahThreadLocalData::disable_plab_retries(thread); 237 } 238 } 239 // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs. 240 } 241 break; 242 } 243 default: { 244 ShouldNotReachHere(); 245 break; 246 } 247 } 248 } 249 250 if (copy == nullptr) { 251 // If we failed to allocate in LAB, we'll try a shared allocation. 252 if (!is_promotion || !has_plab || (size > PLAB::min_size())) { 253 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen, is_promotion); 254 copy = allocate_memory(req); 255 alloc_from_lab = false; 256 } 257 // else, we leave copy equal to nullptr, signaling a promotion failure below if appropriate. 258 // We choose not to promote objects smaller than PLAB::min_size() by way of shared allocations, as this is too 259 // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future 260 // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= PLAB::min_size()) 261 } 262 #ifdef ASSERT 263 } 264 #endif 265 266 if (copy == nullptr) { 267 if (target_gen == OLD_GENERATION) { 268 if (from_region->is_young()) { 269 // Signal that promotion failed. Will evacuate this old object somewhere in young gen. 270 old_generation()->handle_failed_promotion(thread, size); 271 return nullptr; 272 } else { 273 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this 274 // after the evacuation threads have finished. 275 old_generation()->handle_failed_evacuation(); 276 } 277 } 278 279 control_thread()->handle_alloc_failure_evac(size); 280 281 oom_evac_handler()->handle_out_of_memory_during_evacuation(); 282 283 return ShenandoahBarrierSet::resolve_forwarded(p); 284 } 285 286 // Copy the object: 287 evac_tracker()->begin_evacuation(thread, size * HeapWordSize); 288 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size); 289 290 oop copy_val = cast_to_oop(copy); 291 292 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) { 293 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); 294 } 295 296 // Try to install the new forwarding pointer. 297 ContinuationGCSupport::relativize_stack_chunk(copy_val); 298 299 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); 300 if (result == copy_val) { 301 // Successfully evacuated. Our copy is now the public one! 302 evac_tracker()->end_evacuation(thread, size * HeapWordSize); 303 if (target_gen == OLD_GENERATION) { 304 old_generation()->handle_evacuation(copy, size, from_region->is_young()); 305 } else { 306 // When copying to the old generation above, we don't care 307 // about recording object age in the census stats. 308 assert(target_gen == YOUNG_GENERATION, "Error"); 309 // We record this census only when simulating pre-adaptive tenuring behavior, or 310 // when we have been asked to record the census at evacuation rather than at mark 311 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) { 312 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val)); 313 } 314 } 315 shenandoah_assert_correct(nullptr, copy_val); 316 return copy_val; 317 } else { 318 // Failed to evacuate. We need to deal with the object that is left behind. Since this 319 // new allocation is certainly after TAMS, it will be considered live in the next cycle. 320 // But if it happens to contain references to evacuated regions, those references would 321 // not get updated for this stale copy during this cycle, and we will crash while scanning 322 // it the next cycle. 323 if (alloc_from_lab) { 324 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next 325 // object will overwrite this stale copy, or the filler object on LAB retirement will 326 // do this. 327 switch (target_gen) { 328 case YOUNG_GENERATION: { 329 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); 330 break; 331 } 332 case OLD_GENERATION: { 333 ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size); 334 if (is_promotion) { 335 ShenandoahThreadLocalData::subtract_from_plab_promoted(thread, size * HeapWordSize); 336 } else { 337 ShenandoahThreadLocalData::subtract_from_plab_evacuated(thread, size * HeapWordSize); 338 } 339 break; 340 } 341 default: { 342 ShouldNotReachHere(); 343 break; 344 } 345 } 346 } else { 347 // For non-LAB allocations, we have no way to retract the allocation, and 348 // have to explicitly overwrite the copy with the filler object. With that overwrite, 349 // we have to keep the fwdptr initialized and pointing to our (stale) copy. 350 assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size"); 351 fill_with_object(copy, size); 352 shenandoah_assert_correct(nullptr, copy_val); 353 // For non-LAB allocations, the object has already been registered 354 } 355 shenandoah_assert_correct(nullptr, result); 356 return result; 357 } 358 } 359 360 inline HeapWord* ShenandoahGenerationalHeap::allocate_from_plab(Thread* thread, size_t size, bool is_promotion) { 361 assert(UseTLAB, "TLABs should be enabled"); 362 363 PLAB* plab = ShenandoahThreadLocalData::plab(thread); 364 HeapWord* obj; 365 366 if (plab == nullptr) { 367 assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name()); 368 // No PLABs in this thread, fallback to shared allocation 369 return nullptr; 370 } else if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { 371 return nullptr; 372 } 373 // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy 374 obj = plab->allocate(size); 375 if ((obj == nullptr) && (plab->words_remaining() < plab_min_size())) { 376 // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations 377 obj = allocate_from_plab_slow(thread, size, is_promotion); 378 } 379 // if plab->words_remaining() >= ShenGenHeap::heap()->plab_min_size(), just return nullptr so we can use a shared allocation 380 if (obj == nullptr) { 381 return nullptr; 382 } 383 384 if (is_promotion) { 385 ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize); 386 } else { 387 ShenandoahThreadLocalData::add_to_plab_evacuated(thread, size * HeapWordSize); 388 } 389 return obj; 390 } 391 392 // Establish a new PLAB and allocate size HeapWords within it. 393 HeapWord* ShenandoahGenerationalHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) { 394 // New object should fit the PLAB size 395 396 assert(mode()->is_generational(), "PLABs only relevant to generational GC"); 397 const size_t plab_min_size = this->plab_min_size(); 398 const size_t min_size = (size > plab_min_size)? align_up(size, CardTable::card_size_in_words()): plab_min_size; 399 400 // Figure out size of new PLAB, looking back at heuristics. Expand aggressively. PLABs must align on size 401 // of card table in order to avoid the need for synchronization when registering newly allocated objects within 402 // the card table. 403 size_t cur_size = ShenandoahThreadLocalData::plab_size(thread); 404 if (cur_size == 0) { 405 cur_size = plab_min_size; 406 } 407 408 // Limit growth of PLABs to the smaller of ShenandoahMaxEvacLABRatio * the minimum size and ShenandoahHumongousThreshold. 409 // This minimum value is represented by generational_heap->plab_max_size(). Enforcing this limit enables more equitable 410 // distribution of available evacuation budget between the many threads that are coordinating in the evacuation effort. 411 size_t future_size = MIN2(cur_size * 2, plab_max_size()); 412 assert(is_aligned(future_size, CardTable::card_size_in_words()), "Align by design, future_size: " SIZE_FORMAT 413 ", alignment: " SIZE_FORMAT ", cur_size: " SIZE_FORMAT ", max: " SIZE_FORMAT, 414 future_size, (size_t) CardTable::card_size_in_words(), cur_size, plab_max_size()); 415 416 // Record new heuristic value even if we take any shortcut. This captures 417 // the case when moderately-sized objects always take a shortcut. At some point, 418 // heuristics should catch up with them. Note that the requested cur_size may 419 // not be honored, but we remember that this is the preferred size. 420 ShenandoahThreadLocalData::set_plab_size(thread, future_size); 421 if (cur_size < size) { 422 // The PLAB to be allocated is still not large enough to hold the object. Fall back to shared allocation. 423 // This avoids retiring perfectly good PLABs in order to represent a single large object allocation. 424 return nullptr; 425 } 426 427 // Retire current PLAB, and allocate a new one. 428 PLAB* plab = ShenandoahThreadLocalData::plab(thread); 429 if (plab->words_remaining() < plab_min_size) { 430 // Retire current PLAB, and allocate a new one. 431 // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. This 432 // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is 433 // aligned with the start of a card's memory range. 434 retire_plab(plab, thread); 435 436 size_t actual_size = 0; 437 // allocate_new_plab resets plab_evacuated and plab_promoted and disables promotions if old-gen available is 438 // less than the remaining evacuation need. It also adjusts plab_preallocated and expend_promoted if appropriate. 439 HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size); 440 if (plab_buf == nullptr) { 441 if (min_size == plab_min_size) { 442 // Disable plab promotions for this thread because we cannot even allocate a plab of minimal size. This allows us 443 // to fail faster on subsequent promotion attempts. 444 ShenandoahThreadLocalData::disable_plab_promotions(thread); 445 } 446 return nullptr; 447 } else { 448 ShenandoahThreadLocalData::enable_plab_retries(thread); 449 } 450 // Since the allocated PLAB may have been down-sized for alignment, plab->allocate(size) below may still fail. 451 if (ZeroTLAB) { 452 // ... and clear it. 453 Copy::zero_to_words(plab_buf, actual_size); 454 } else { 455 // ...and zap just allocated object. 456 #ifdef ASSERT 457 // Skip mangling the space corresponding to the object header to 458 // ensure that the returned space is not considered parsable by 459 // any concurrent GC thread. 460 size_t hdr_size = oopDesc::header_size(); 461 Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal); 462 #endif // ASSERT 463 } 464 assert(is_aligned(actual_size, CardTable::card_size_in_words()), "Align by design"); 465 plab->set_buf(plab_buf, actual_size); 466 if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { 467 return nullptr; 468 } 469 return plab->allocate(size); 470 } else { 471 // If there's still at least min_size() words available within the current plab, don't retire it. Let's gnaw 472 // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request 473 // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we 474 // reduce the likelihood of evacuation failures, and we reduce the need for downsizing our PLABs. 475 return nullptr; 476 } 477 } 478 479 HeapWord* ShenandoahGenerationalHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) { 480 // Align requested sizes to card-sized multiples. Align down so that we don't violate max size of TLAB. 481 assert(is_aligned(min_size, CardTable::card_size_in_words()), "Align by design"); 482 assert(word_size >= min_size, "Requested PLAB is too small"); 483 484 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); 485 // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread 486 // if we are at risk of infringing on the old-gen evacuation budget. 487 HeapWord* res = allocate_memory(req); 488 if (res != nullptr) { 489 *actual_size = req.actual_size(); 490 } else { 491 *actual_size = 0; 492 } 493 assert(is_aligned(res, CardTable::card_size_in_words()), "Align by design"); 494 return res; 495 } 496 497 // TODO: It is probably most efficient to register all objects (both promotions and evacuations) that were allocated within 498 // this plab at the time we retire the plab. A tight registration loop will run within both code and data caches. This change 499 // would allow smaller and faster in-line implementation of alloc_from_plab(). Since plabs are aligned on card-table boundaries, 500 // this object registration loop can be performed without acquiring a lock. 501 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab, Thread* thread) { 502 // We don't enforce limits on plab_evacuated. We let it consume all available old-gen memory in order to reduce 503 // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion 504 // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any 505 // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle. 506 507 // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to 508 // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions. 509 // 1. Some of the plab may have been dedicated to evacuations. 510 // 2. Some of the plab may have been abandoned due to waste (at the end of the plab). 511 size_t not_promoted = 512 ShenandoahThreadLocalData::get_plab_preallocated_promoted(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread); 513 ShenandoahThreadLocalData::reset_plab_promoted(thread); 514 ShenandoahThreadLocalData::reset_plab_evacuated(thread); 515 ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); 516 if (not_promoted > 0) { 517 old_generation()->unexpend_promoted(not_promoted); 518 } 519 const size_t original_waste = plab->waste(); 520 HeapWord* const top = plab->top(); 521 522 // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable. 523 // It adds the size of this unused memory, in words, to plab->waste(). 524 plab->retire(); 525 if (top != nullptr && plab->waste() > original_waste && is_in_old(top)) { 526 // If retiring the plab created a filler object, then we need to register it with our card scanner so it can 527 // safely walk the region backing the plab. 528 log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, 529 plab->waste() - original_waste, p2i(top)); 530 card_scan()->register_object_without_lock(top); 531 } 532 } 533 534 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) { 535 Thread* thread = Thread::current(); 536 retire_plab(plab, thread); 537 } 538 539 ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalHeap::balance_generations() { 540 shenandoah_assert_heaplocked_or_safepoint(); 541 542 ShenandoahOldGeneration* old_gen = old_generation(); 543 const ssize_t old_region_balance = old_gen->get_region_balance(); 544 old_gen->set_region_balance(0); 545 546 if (old_region_balance > 0) { 547 const auto old_region_surplus = checked_cast<size_t>(old_region_balance); 548 const bool success = generation_sizer()->transfer_to_young(old_region_surplus); 549 return TransferResult { 550 success, old_region_surplus, "young" 551 }; 552 } 553 554 if (old_region_balance < 0) { 555 const auto old_region_deficit = checked_cast<size_t>(-old_region_balance); 556 const bool success = generation_sizer()->transfer_to_old(old_region_deficit); 557 if (!success) { 558 old_gen->handle_failed_transfer(); 559 } 560 return TransferResult { 561 success, old_region_deficit, "old" 562 }; 563 } 564 565 return TransferResult {true, 0, "none"}; 566 } 567 568 // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations 569 // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to 570 // xfer_limit, and any surplus is transferred to the young generation. 571 // xfer_limit is the maximum we're able to transfer from young to old. 572 void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions) { 573 574 // We can limit the old reserve to the size of anticipated promotions: 575 // max_old_reserve is an upper bound on memory evacuated from old and promoted to old, 576 // clamped by the old generation space available. 577 // 578 // Here's the algebra. 579 // Let SOEP = ShenandoahOldEvacRatioPercent, 580 // OE = old evac, 581 // YE = young evac, and 582 // TE = total evac = OE + YE 583 // By definition: 584 // SOEP/100 = OE/TE 585 // = OE/(OE+YE) 586 // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c) 587 // = OE/YE 588 // => OE = YE*SOEP/(100-SOEP) 589 590 // We have to be careful in the event that SOEP is set to 100 by the user. 591 assert(ShenandoahOldEvacRatioPercent <= 100, "Error"); 592 const size_t old_available = old_generation()->available(); 593 // The free set will reserve this amount of memory to hold young evacuations 594 const size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100; 595 596 // In the case that ShenandoahOldEvacRatioPercent equals 100, max_old_reserve is limited only by xfer_limit. 597 598 const size_t bound_on_old_reserve = old_available + old_xfer_limit + young_reserve; 599 const size_t max_old_reserve = (ShenandoahOldEvacRatioPercent == 100)? 600 bound_on_old_reserve: MIN2((young_reserve * ShenandoahOldEvacRatioPercent) / (100 - ShenandoahOldEvacRatioPercent), 601 bound_on_old_reserve); 602 603 const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); 604 605 // Decide how much old space we should reserve for a mixed collection 606 size_t reserve_for_mixed = 0; 607 if (old_generation()->has_unprocessed_collection_candidates()) { 608 // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we 609 // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation. 610 const size_t max_evac_need = (size_t) 611 (old_generation()->unprocessed_collection_candidates_live_memory() * ShenandoahOldEvacWaste); 612 assert(old_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes, 613 "Unaffiliated available must be less than total available"); 614 const size_t old_fragmented_available = 615 old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes; 616 reserve_for_mixed = max_evac_need + old_fragmented_available; 617 if (reserve_for_mixed > max_old_reserve) { 618 reserve_for_mixed = max_old_reserve; 619 } 620 } 621 622 // Decide how much space we should reserve for promotions from young 623 size_t reserve_for_promo = 0; 624 const size_t promo_load = old_generation()->get_promotion_potential(); 625 const bool doing_promotions = promo_load > 0; 626 if (doing_promotions) { 627 // We're promoting and have a bound on the maximum amount that can be promoted 628 assert(max_old_reserve >= reserve_for_mixed, "Sanity"); 629 const size_t available_for_promotions = max_old_reserve - reserve_for_mixed; 630 reserve_for_promo = MIN2((size_t)(promo_load * ShenandoahPromoEvacWaste), available_for_promotions); 631 } 632 633 // This is the total old we want to ideally reserve 634 const size_t old_reserve = reserve_for_mixed + reserve_for_promo; 635 assert(old_reserve <= max_old_reserve, "cannot reserve more than max for old evacuations"); 636 637 // We now check if the old generation is running a surplus or a deficit. 638 const size_t max_old_available = old_generation()->available() + old_cset_regions * region_size_bytes; 639 if (max_old_available >= old_reserve) { 640 // We are running a surplus, so the old region surplus can go to young 641 const size_t old_surplus = (max_old_available - old_reserve) / region_size_bytes; 642 const size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_cset_regions; 643 const size_t old_region_surplus = MIN2(old_surplus, unaffiliated_old_regions); 644 old_generation()->set_region_balance(checked_cast<ssize_t>(old_region_surplus)); 645 } else { 646 // We are running a deficit which we'd like to fill from young. 647 // Ignore that this will directly impact young_generation()->max_capacity(), 648 // indirectly impacting young_reserve and old_reserve. These computations are conservative. 649 // Note that deficit is rounded up by one region. 650 const size_t old_need = (old_reserve - max_old_available + region_size_bytes - 1) / region_size_bytes; 651 const size_t max_old_region_xfer = old_xfer_limit / region_size_bytes; 652 653 // Round down the regions we can transfer from young to old. If we're running short 654 // on young-gen memory, we restrict the xfer. Old-gen collection activities will be 655 // curtailed if the budget is restricted. 656 const size_t old_region_deficit = MIN2(old_need, max_old_region_xfer); 657 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit)); 658 } 659 } 660 661 void ShenandoahGenerationalHeap::reset_generation_reserves() { 662 young_generation()->set_evacuation_reserve(0); 663 old_generation()->set_evacuation_reserve(0); 664 old_generation()->set_promoted_reserve(0); 665 } 666 667 void ShenandoahGenerationalHeap::TransferResult::print_on(const char* when, outputStream* ss) const { 668 auto heap = ShenandoahGenerationalHeap::heap(); 669 ShenandoahYoungGeneration* const young_gen = heap->young_generation(); 670 ShenandoahOldGeneration* const old_gen = heap->old_generation(); 671 const size_t young_available = young_gen->available(); 672 const size_t old_available = old_gen->available(); 673 ss->print_cr("After %s, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " 674 PROPERFMT ", young_available: " PROPERFMT, 675 when, 676 success? "successfully transferred": "failed to transfer", region_count, region_destination, 677 PROPERFMTARGS(old_available), PROPERFMTARGS(young_available)); 678 } 679 680 void ShenandoahGenerationalHeap::coalesce_and_fill_old_regions(bool concurrent) { 681 class ShenandoahGlobalCoalesceAndFill : public WorkerTask { 682 private: 683 ShenandoahPhaseTimings::Phase _phase; 684 ShenandoahRegionIterator _regions; 685 public: 686 explicit ShenandoahGlobalCoalesceAndFill(ShenandoahPhaseTimings::Phase phase) : 687 WorkerTask("Shenandoah Global Coalesce"), 688 _phase(phase) {} 689 690 void work(uint worker_id) override { 691 ShenandoahWorkerTimingsTracker timer(_phase, 692 ShenandoahPhaseTimings::ScanClusters, 693 worker_id, true); 694 ShenandoahHeapRegion* region; 695 while ((region = _regions.next()) != nullptr) { 696 // old region is not in the collection set and was not immediately trashed 697 if (region->is_old() && region->is_active() && !region->is_humongous()) { 698 // Reset the coalesce and fill boundary because this is a global collect 699 // and cannot be preempted by young collects. We want to be sure the entire 700 // region is coalesced here and does not resume from a previously interrupted 701 // or completed coalescing. 702 region->begin_preemptible_coalesce_and_fill(); 703 region->oop_coalesce_and_fill(false); 704 } 705 } 706 } 707 }; 708 709 ShenandoahPhaseTimings::Phase phase = concurrent ? 710 ShenandoahPhaseTimings::conc_coalesce_and_fill : 711 ShenandoahPhaseTimings::degen_gc_coalesce_and_fill; 712 713 // This is not cancellable 714 ShenandoahGlobalCoalesceAndFill coalesce(phase); 715 workers()->run_task(&coalesce); 716 old_generation()->set_parseable(true); 717 } 718