1 /*
2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
3 * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shenandoah/shenandoahAgeCensus.hpp"
27 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahFreeSet.hpp"
30 #include "gc/shenandoah/shenandoahGeneration.hpp"
31 #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp"
32 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
33 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
37 #include "gc/shenandoah/shenandoahInitLogger.hpp"
38 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
39 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
40 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
41 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
42 #include "gc/shenandoah/shenandoahRegulatorThread.hpp"
43 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
44 #include "gc/shenandoah/shenandoahUtils.hpp"
45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
46 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
47 #include "logging/log.hpp"
48 #include "utilities/events.hpp"
49
50
51 class ShenandoahGenerationalInitLogger : public ShenandoahInitLogger {
52 public:
53 static void print() {
54 ShenandoahGenerationalInitLogger logger;
55 logger.print_all();
56 }
57 protected:
58 void print_gc_specific() override {
59 ShenandoahInitLogger::print_gc_specific();
60
61 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
62 log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name());
63 log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name());
64 }
65 };
66
67 size_t ShenandoahGenerationalHeap::calculate_min_plab() {
68 return PLAB::min_size();
69 }
70
71 size_t ShenandoahGenerationalHeap::calculate_max_plab() {
72 return ShenandoahHeapRegion::max_tlab_size_words();
73 }
74
75 // Returns size in bytes
76 size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc() const {
77 return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available());
78 }
79
80 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
81 ShenandoahHeap(policy),
82 _age_census(nullptr),
83 _min_plab_size(calculate_min_plab()),
84 _max_plab_size(calculate_max_plab()),
85 _regulator_thread(nullptr),
86 _young_gen_memory_pool(nullptr),
87 _old_gen_memory_pool(nullptr) {
88 }
89
90 void ShenandoahGenerationalHeap::initialize_generations() {
91 ShenandoahHeap::initialize_generations();
92 _young_generation->post_initialize(this);
93 _old_generation->post_initialize(this);
94 }
95
96 void ShenandoahGenerationalHeap::post_initialize() {
97 ShenandoahHeap::post_initialize();
98 _age_census = new ShenandoahAgeCensus();
99 }
100
101 void ShenandoahGenerationalHeap::post_initialize_heuristics() {
102 ShenandoahHeap::post_initialize_heuristics();
103 _young_generation->post_initialize_heuristics();
104 _old_generation->post_initialize_heuristics();
105 }
106
107 void ShenandoahGenerationalHeap::print_init_logger() const {
108 ShenandoahGenerationalInitLogger logger;
109 logger.print_all();
110 }
111
112 void ShenandoahGenerationalHeap::initialize_heuristics() {
113 // Initialize global generation and heuristics even in generational mode.
114 ShenandoahHeap::initialize_heuristics();
115
116 _young_generation = new ShenandoahYoungGeneration(max_workers());
117 _old_generation = new ShenandoahOldGeneration(max_workers());
118 _young_generation->initialize_heuristics(mode());
119 _old_generation->initialize_heuristics(mode());
120 }
121
122 void ShenandoahGenerationalHeap::initialize_serviceability() {
123 assert(mode()->is_generational(), "Only for the generational mode");
124 _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this);
125 _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this);
126 cycle_memory_manager()->add_pool(_young_gen_memory_pool);
127 cycle_memory_manager()->add_pool(_old_gen_memory_pool);
128 stw_memory_manager()->add_pool(_young_gen_memory_pool);
129 stw_memory_manager()->add_pool(_old_gen_memory_pool);
130 }
131
132 GrowableArray<MemoryPool*> ShenandoahGenerationalHeap::memory_pools() {
133 assert(mode()->is_generational(), "Only for the generational mode");
134 GrowableArray<MemoryPool*> memory_pools(2);
135 memory_pools.append(_young_gen_memory_pool);
136 memory_pools.append(_old_gen_memory_pool);
137 return memory_pools;
138 }
139
140 void ShenandoahGenerationalHeap::initialize_controller() {
141 auto control_thread = new ShenandoahGenerationalControlThread();
142 _control_thread = control_thread;
143 _regulator_thread = new ShenandoahRegulatorThread(control_thread);
144 }
145
146 void ShenandoahGenerationalHeap::gc_threads_do(ThreadClosure* tcl) const {
147 if (!shenandoah_policy()->is_at_shutdown()) {
148 ShenandoahHeap::gc_threads_do(tcl);
149 tcl->do_thread(regulator_thread());
150 }
151 }
152
153 void ShenandoahGenerationalHeap::stop() {
154 ShenandoahHeap::stop();
155 regulator_thread()->stop();
156 }
157
158 void ShenandoahGenerationalHeap::start_idle_span() {
159 young_generation()->heuristics()->start_idle_span();
160 }
161
162 bool ShenandoahGenerationalHeap::requires_barriers(stackChunkOop obj) const {
163 if (is_idle()) {
164 return false;
165 }
166
167 if (is_concurrent_young_mark_in_progress() && is_in_young(obj) && !marking_context()->allocated_after_mark_start(obj)) {
168 // We are marking young, this object is in young, and it is below the TAMS
169 return true;
170 }
171
172 if (is_in_old(obj)) {
173 // Card marking barriers are required for objects in the old generation
174 return true;
175 }
176
177 if (has_forwarded_objects()) {
178 // Object may have pointers that need to be updated
179 return true;
180 }
181
182 return false;
183 }
184
185 void ShenandoahGenerationalHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
186 ShenandoahRegionIterator regions;
187 ShenandoahGenerationalEvacuationTask task(this, generation, ®ions, concurrent, false /* only promote regions */);
188 workers()->run_task(&task);
189 }
190
191 void ShenandoahGenerationalHeap::promote_regions_in_place(ShenandoahGeneration* generation, bool concurrent) {
192 ShenandoahRegionIterator regions;
193 ShenandoahGenerationalEvacuationTask task(this, generation, ®ions, concurrent, true /* only promote regions */);
194 workers()->run_task(&task);
195 }
196
197 oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) {
198 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
199
200 ShenandoahHeapRegion* from_region = heap_region_containing(p);
201 assert(!from_region->is_humongous(), "never evacuate humongous objects");
202
203 // Try to keep the object in the same generation
204 const ShenandoahAffiliation target_gen = from_region->affiliation();
205
206 if (target_gen == YOUNG_GENERATION) {
207 markWord mark = p->mark();
208 if (mark.is_marked()) {
209 // Already forwarded.
210 return ShenandoahBarrierSet::resolve_forwarded(p);
211 }
212
213 if (mark.has_displaced_mark_helper()) {
214 // We don't want to deal with MT here just to ensure we read the right mark word.
215 // Skip the potential promotion attempt for this one.
216 } else if (age_census()->is_tenurable(from_region->age() + mark.age())) {
217 // If the object is tenurable, try to promote it
218 oop result = try_evacuate_object<YOUNG_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
219
220 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
221 if (result != nullptr) {
222 return result;
223 }
224 }
225 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
226 }
227
228 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
229 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
230 }
231
232 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
233 // to OLD_GENERATION.
234 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
235 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
236 bool alloc_from_lab = true;
237 bool has_plab = false;
238 HeapWord* copy = nullptr;
239
240 markWord mark = p->mark();
241 if (mark.is_forwarded()) {
242 return ShenandoahForwarding::get_forwardee(p);
243 }
244 size_t old_size = ShenandoahForwarding::size(p);
245 size_t size = p->copy_size(old_size, mark);
246
247 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
248
249 #ifdef ASSERT
250 if (ShenandoahOOMDuringEvacALot &&
251 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
252 copy = nullptr;
253 } else {
254 #endif
255 if (UseTLAB) {
256 switch (TO_GENERATION) {
257 case YOUNG_GENERATION: {
258 copy = allocate_from_gclab(thread, size);
259 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
260 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
261 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
262 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
263 copy = allocate_from_gclab(thread, size);
264 // If we still get nullptr, we'll try a shared allocation below.
265 }
266 break;
267 }
268 case OLD_GENERATION: {
269 ShenandoahPLAB* shenandoah_plab = ShenandoahThreadLocalData::shenandoah_plab(thread);
270 if (shenandoah_plab != nullptr) {
271 has_plab = true;
272 copy = shenandoah_plab->allocate(size, is_promotion);
273 if (copy == nullptr && size < shenandoah_plab->desired_size() && shenandoah_plab->retries_enabled()) {
274 // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because
275 // the requested object does not fit within the current plab but the plab still has an "abundance" of memory,
276 // where abundance is defined as >= ShenGenHeap::plab_min_size(). In the former case, we try shrinking the
277 // desired PLAB size to the minimum and retry PLAB allocation to avoid cascading of shared memory allocations.
278 // Shrinking the desired PLAB size may allow us to eke out a small PLAB while staying beneath evacuation reserve.
279 if (shenandoah_plab->plab()->words_remaining() < plab_min_size()) {
280 shenandoah_plab->set_desired_size(plab_min_size());
281 copy = shenandoah_plab->allocate(size, is_promotion);
282 if (copy == nullptr) {
283 // If we still get nullptr, we'll try a shared allocation below.
284 // However, don't continue to retry until we have success (probably in next GC pass)
285 shenandoah_plab->disable_retries();
286 }
287 }
288 }
289 }
290 break;
291 }
292 default: {
293 ShouldNotReachHere();
294 break;
295 }
296 }
297 }
298
299 if (copy == nullptr) {
300 // If we failed to allocate in LAB, we'll try a shared allocation.
301 if (!is_promotion || !has_plab || (size > PLAB::min_size())) {
302 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, TO_GENERATION, is_promotion);
303 copy = allocate_memory(req);
304 alloc_from_lab = false;
305 }
306 // else, we leave copy equal to nullptr, signaling a promotion failure below if appropriate.
307 // We choose not to promote objects smaller than size_threshold by way of shared allocations as this is too
308 // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future
309 // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= size_threshhold).
310 }
311 #ifdef ASSERT
312 }
313 #endif
314
315 if (copy == nullptr) {
316 if (TO_GENERATION == OLD_GENERATION) {
317 if (FROM_GENERATION == YOUNG_GENERATION) {
318 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
319 old_generation()->handle_failed_promotion(thread, size);
320 return nullptr;
321 } else {
322 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
323 // after the evacuation threads have finished.
324 old_generation()->handle_failed_evacuation();
325 }
326 }
327
328 control_thread()->handle_alloc_failure_evac(size);
329
330 // Install the self-forwarded bit so other evacuators/LRBs see the
331 // object as "already handled, do not try to evacuate". The CAS may
332 // fail if another thread concurrently installed a real forwardee or
333 // self-forwarded first.
334 markWord old_mark = p->mark();
335 if (old_mark.is_forwarded()) {
336 return ShenandoahForwarding::get_forwardee(p);
337 }
338 oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
339 if (winner == nullptr) {
340 // We own the self-forwarding. Flag the from-region so the degen/full
341 // GC entry drain knows to scan it for self_fwd bits to clear.
342 heap_region_containing(p)->set_has_self_forwards();
343 return p;
344 }
345 return winner;
346 }
347
348 if (ShenandoahEvacTracking) {
349 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
350 }
351
352 // Copy the object:
353 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
354 oop copy_val = cast_to_oop(copy);
355
356 // Initialize the identity hash on the copy before installing the forwarding
357 // pointer, using the mark word we captured earlier. We must do this before
358 // the CAS so that the copy is fully initialized when it becomes visible to
359 // other threads. Using the captured mark (rather than re-reading the copy's
360 // mark) avoids races with other threads that may have evacuated p and
361 // installed a forwarding pointer in the meantime.
362 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
363 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
364 }
365
366 // Update the age of the evacuated object
367 if (TO_GENERATION == YOUNG_GENERATION) {
368 increase_object_age(copy_val, from_region_age + 1);
369 }
370
371 // Relativize stack chunks before publishing the copy. After the forwarding CAS,
372 // mutators can see the copy and thaw it via the fast path if flags == 0. We must
373 // relativize derived pointers and set gc_mode before that happens. Skip if the
374 // copy's mark word is already a forwarding pointer (another thread won the race
375 // and overwrote the original's header before we copied it).
376 if (!ShenandoahForwarding::is_forwarded(copy_val)) {
377 ContinuationGCSupport::relativize_stack_chunk(copy_val);
378 }
379
380 // Try to install the new forwarding pointer.
381 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
382 if (result == copy_val) {
383 // Successfully evacuated. Our copy is now the public one!
384 if (ShenandoahEvacTracking) {
385 // Record that the evacuation succeeded
386 evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
387 }
388 } else {
389 // Failed to evacuate. We need to deal with the object that is left behind. Since this
390 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
391 // But if it happens to contain references to evacuated regions, those references would
392 // not get updated for this stale copy during this cycle, and we will crash while scanning
393 // it the next cycle.
394 if (alloc_from_lab) {
395 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
396 // object will overwrite this stale copy, or the filler object on LAB retirement will
397 // do this.
398 switch (TO_GENERATION) {
399 case YOUNG_GENERATION: {
400 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
401 break;
402 }
403 case OLD_GENERATION: {
404 ShenandoahThreadLocalData::shenandoah_plab(thread)->plab()->undo_allocation(copy, size);
405 if (is_promotion) {
406 ShenandoahThreadLocalData::shenandoah_plab(thread)->subtract_from_promoted(size * HeapWordSize);
407 }
408 break;
409 }
410 default: {
411 ShouldNotReachHere();
412 break;
413 }
414 }
415 } else {
416 // For non-LAB allocations, we have no way to retract the allocation, and
417 // have to explicitly overwrite the copy with the filler object. With that overwrite,
418 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
419 assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
420 fill_with_object(copy, size);
421 }
422 }
423 shenandoah_assert_correct(nullptr, result);
424 return result;
425 }
426
427 template oop ShenandoahGenerationalHeap::try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(oop p, Thread* thread, uint from_region_age);
428 template oop ShenandoahGenerationalHeap::try_evacuate_object<YOUNG_GENERATION, OLD_GENERATION>(oop p, Thread* thread, uint from_region_age);
429 template oop ShenandoahGenerationalHeap::try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(oop p, Thread* thread, uint from_region_age);
430
431 // Call this function at the end of a GC cycle in order to establish proper sizes of young and old reserves,
432 // setting the old-generation balance so that GC can perform the anticipated evacuations.
433 //
434 // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations
435 // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to
436 // mutator_xfer_limit, and any surplus is transferred to the young generation. mutator_xfer_limit is
437 // the maximum we're able to transfer from young to old. The mutator_xfer_limit constrains the transfer
438 // of memory from young to old. It does not limit young reserves.
439 void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t mutator_xfer_limit,
440 size_t old_trashed_regions, size_t young_trashed_regions) {
441 shenandoah_assert_heaplocked();
442 // We can limit the old reserve to the size of anticipated promotions:
443 // max_old_reserve is an upper bound on memory evacuated from old and promoted to old,
444 // clamped by the old generation space available.
445 //
446 // Here's the algebra.
447 // Let SOEP = ShenandoahOldEvacPercent,
448 // OE = old evac,
449 // YE = young evac, and
450 // TE = total evac = OE + YE
451 // By definition:
452 // SOEP/100 = OE/TE
453 // = OE/(OE+YE)
454 // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
455 // = OE/YE
456 // => OE = YE*SOEP/(100-SOEP)
457
458 // We have to be careful in the event that SOEP is set to 100 by the user.
459 assert(ShenandoahOldEvacPercent <= 100, "Error");
460 const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
461
462 ShenandoahOldGeneration* old_gen = old_generation();
463 size_t old_capacity = old_gen->max_capacity();
464 size_t old_usage = old_gen->used(); // includes humongous waste
465 size_t old_currently_available =
466 ((old_capacity >= old_usage)? old_capacity - old_usage: 0) + old_trashed_regions * region_size_bytes;
467
468 ShenandoahYoungGeneration* young_gen = young_generation();
469 size_t young_capacity = young_gen->max_capacity();
470 size_t young_usage = young_gen->used(); // includes humongous waste
471 size_t young_available = ((young_capacity >= young_usage)? young_capacity - young_usage: 0);
472 size_t freeset_available = free_set()->available_locked();
473 if (young_available > freeset_available) {
474 young_available = freeset_available;
475 }
476 young_available += young_trashed_regions * region_size_bytes;
477
478 // The free set will reserve this amount of memory to hold young evacuations (initialized to the ideal reserve)
479 size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100;
480
481 // If ShenandoahOldEvacPercent equals 100, max_old_reserve is limited only by mutator_xfer_limit and young_reserve
482 const size_t bound_on_old_reserve =
483 ((old_currently_available + mutator_xfer_limit + young_reserve) * ShenandoahOldEvacPercent) / 100;
484 size_t proposed_max_old = ((ShenandoahOldEvacPercent == 100)?
485 bound_on_old_reserve:
486 MIN2((young_reserve * ShenandoahOldEvacPercent) / (100 - ShenandoahOldEvacPercent),
487 bound_on_old_reserve));
488 assert(mutator_xfer_limit <= young_available,
489 "Cannot transfer (%zu) memory that is not available (%zu)", mutator_xfer_limit, young_available);
490
491 if (young_reserve > young_available) {
492 young_reserve = young_available;
493 }
494 // We allow young_reserve to exceed mutator_xfer_limit. Essentially, this means the GC is already behind the pace
495 // of mutator allocations, and we'll need to trigger the next GC as soon as possible.
496 if (mutator_xfer_limit > young_reserve) {
497 mutator_xfer_limit -= young_reserve;
498 } else {
499 mutator_xfer_limit = 0;
500 }
501
502 // Decide how much old space we should reserve for a mixed collection
503 size_t proposed_reserve_for_mixed = 0;
504 const size_t old_fragmented_available =
505 old_currently_available - (old_generation()->free_unaffiliated_regions() + old_trashed_regions) * region_size_bytes;
506
507 if (old_fragmented_available > proposed_max_old) {
508 // In this case, the old_fragmented_available is greater than the desired amount of evacuation to old.
509 // We'll use all of this memory to hold results of old evacuation, and we'll give back to the young generation
510 // any old regions that are not fragmented.
511 //
512 // This scenario may happen after we have promoted many regions in place, and each of these regions had non-zero
513 // unused memory, so there is now an abundance of old-fragmented available memory, even more than the desired
514 // percentage for old reserve. We cannot transfer these fragmented regions back to young. Instead we make the
515 // best of the situation by using this fragmented memory for both promotions and evacuations.
516
517 proposed_max_old = old_fragmented_available;
518 }
519 // Otherwise: old_fragmented_available <= proposed_max_old. Do not shrink proposed_max_old from the original computation.
520
521 // Though we initially set proposed_reserve_for_promo to equal the entirety of old fragmented available, we have the
522 // opportunity below to shift some of this memory into the proposed_reserve_for_mixed.
523 size_t proposed_reserve_for_promo = old_fragmented_available;
524 const size_t max_old_reserve = proposed_max_old;
525
526 const size_t mixed_candidate_live_memory = old_generation()->unprocessed_collection_candidates_live_memory();
527 const bool doing_mixed = (mixed_candidate_live_memory > 0);
528 if (doing_mixed) {
529 // In the ideal, all of the memory reserved for mixed evacuation would be unfragmented, but we don't enforce
530 // this. Note that the initial value of max_evac_need is conservative because we may not evacuate all of the
531 // remaining mixed evacuation candidates in a single cycle.
532 const size_t max_evac_need = (size_t) (mixed_candidate_live_memory * ShenandoahOldEvacWaste);
533 assert(old_currently_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes,
534 "Unaffiliated available must be less than total available");
535
536 // We prefer to evacuate all of mixed into unfragmented memory, and will expand old in order to do so, unless
537 // we already have too much fragmented available memory in old.
538 proposed_reserve_for_mixed = max_evac_need;
539 if (proposed_reserve_for_mixed + proposed_reserve_for_promo > max_old_reserve) {
540 // We're trying to reserve more memory than is available. So we need to shrink our reserves.
541 size_t excess_reserves = (proposed_reserve_for_mixed + proposed_reserve_for_promo) - max_old_reserve;
542 // We need to shrink reserves by excess_reserves. We prefer to shrink by reducing promotion, giving priority to mixed
543 // evacuation. If the promotion reserve is larger than the amount we need to shrink by, do all the shrinkage there.
544 if (proposed_reserve_for_promo > excess_reserves) {
545 proposed_reserve_for_promo -= excess_reserves;
546 } else {
547 // Otherwise, we'll shrink promotion reserve to zero and we'll shrink the mixed-evac reserve by the remaining excess.
548 excess_reserves -= proposed_reserve_for_promo;
549 proposed_reserve_for_promo = 0;
550 proposed_reserve_for_mixed -= excess_reserves;
551 }
552 }
553 }
554 assert(proposed_reserve_for_mixed + proposed_reserve_for_promo <= max_old_reserve,
555 "Reserve for mixed (%zu) plus reserve for promotions (%zu) must be less than maximum old reserve (%zu)",
556 proposed_reserve_for_mixed, proposed_reserve_for_promo, max_old_reserve);
557
558 // Decide how much additional space we should reserve for promotions from young. We give priority to mixed evacations
559 // over promotions.
560 const size_t promo_load = old_generation()->get_promotion_potential();
561 const bool doing_promotions = promo_load > 0;
562
563 // promo_load represents the combined total of live memory within regions that have reached tenure age. The true
564 // promotion potential is larger than this, because individual objects within regions that have not yet reached tenure
565 // age may be promotable. On the other hand, some of the objects that we intend to promote in the next GC cycle may
566 // die before they are next marked. In the future, the promo_load will include the total size of tenurable objects
567 // residing in regions that have not yet reached tenure age.
568
569 if (doing_promotions) {
570 // We are always doing promotions, even when old_generation->get_promotion_potential() returns 0. As currently implemented,
571 // get_promotion_potential() only knows the total live memory contained within young-generation regions whose age is
572 // tenurable. It does not know whether that memory will still be live at the end of the next mark cycle, and it doesn't
573 // know how much memory is contained within objects whose individual ages are tenurable, which reside in regions with
574 // non-tenurable age. We use this, as adjusted by ShenandoahPromoEvacWaste, as an approximation of the total amount of
575 // memory to be promoted. In the near future, we expect to implement a change that will allow get_promotion_potential()
576 // to account also for the total memory contained within individual objects that are tenure-ready even when they do
577 // not reside in aged regions. This will represent a conservative over approximation of promotable memory because
578 // some of these objects may die before the next GC cycle executes.
579
580 // Be careful not to ask for too much promotion reserves. We have observed jtreg test failures under which a greedy
581 // promotion reserve causes a humongous allocation which is awaiting a full GC to fail (specifically
582 // gc/TestAllocHumongousFragment.java). This happens if too much of the memory reclaimed by the full GC
583 // is immediately reserved so that it cannot be allocated by the waiting mutator. It's not clear that this
584 // particular test is representative of the needs of typical GenShen users. It is really a test of high frequency
585 // Full GCs under heap fragmentation stress.
586
587 size_t promo_need = (size_t) (promo_load * ShenandoahPromoEvacWaste);
588 if (promo_need > proposed_reserve_for_promo) {
589 const size_t available_for_additional_promotions =
590 max_old_reserve - (proposed_reserve_for_mixed + proposed_reserve_for_promo);
591 if (proposed_reserve_for_promo + available_for_additional_promotions >= promo_need) {
592 proposed_reserve_for_promo = promo_need;
593 } else {
594 proposed_reserve_for_promo += available_for_additional_promotions;
595 }
596 }
597 }
598 // else, leave proposed_reserve_for_promo as is. By default, it is initialized to represent old_fragmented_available.
599
600 // This is the total old we want to reserve (initialized to the ideal reserve)
601 size_t proposed_old_reserve = proposed_reserve_for_mixed + proposed_reserve_for_promo;
602
603 // We now check if the old generation is running a surplus or a deficit.
604 size_t old_region_deficit = 0;
605 size_t old_region_surplus = 0;
606
607 size_t mutator_region_xfer_limit = mutator_xfer_limit / region_size_bytes;
608 // align the mutator_xfer_limit on region size
609 mutator_xfer_limit = mutator_region_xfer_limit * region_size_bytes;
610
611 if (old_currently_available >= proposed_old_reserve) {
612 // We are running a surplus, so the old region surplus can go to young
613 const size_t old_surplus = old_currently_available - proposed_old_reserve;
614 old_region_surplus = old_surplus / region_size_bytes;
615 const size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_trashed_regions;
616 old_region_surplus = MIN2(old_region_surplus, unaffiliated_old_regions);
617 old_generation()->set_region_balance(checked_cast<ssize_t>(old_region_surplus));
618 old_currently_available -= old_region_surplus * region_size_bytes;
619 young_available += old_region_surplus * region_size_bytes;
620 } else if (old_currently_available + mutator_xfer_limit >= proposed_old_reserve) {
621 // We know that old_currently_available < proposed_old_reserve because above test failed. Expand old_currently_available.
622 // Mutator's xfer limit is sufficient to satisfy our need: transfer all memory from there.
623 size_t old_deficit = proposed_old_reserve - old_currently_available;
624 old_region_deficit = (old_deficit + region_size_bytes - 1) / region_size_bytes;
625 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit));
626 old_currently_available += old_region_deficit * region_size_bytes;
627 young_available -= old_region_deficit * region_size_bytes;
628 } else {
629 // We know that (old_currently_available < proposed_old_reserve) and
630 // (old_currently_available + mutator_xfer_limit < proposed_old_reserve) because above tests failed.
631 // We need to shrink proposed_old_reserves.
632
633 // We could potentially shrink young_reserves in order to further expand proposed_old_reserves. Let's not bother. The
634 // important thing is that we keep a total amount of memory in reserve in preparation for the next GC cycle. At
635 // the time we choose the next collection set, we'll have an opportunity to shift some of these young reserves
636 // into old reserves if that makes sense.
637
638 // Start by taking all of mutator_xfer_limit into old_currently_available.
639 size_t old_region_deficit = mutator_region_xfer_limit;
640 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit));
641 old_currently_available += old_region_deficit * region_size_bytes;
642 young_available -= old_region_deficit * region_size_bytes;
643
644 assert(old_currently_available < proposed_old_reserve,
645 "Old currently available (%zu) must be less than old reserve (%zu)", old_currently_available, proposed_old_reserve);
646
647 // There's not enough memory to satisfy our desire. Scale back our old-gen intentions. We prefer to satisfy
648 // the budget_overrun entirely from the promotion reserve, if that is large enough. Otherwise, we'll satisfy
649 // the overrun from a combination of promotion and mixed-evacuation reserves.
650 size_t budget_overrun = proposed_old_reserve - old_currently_available;
651 if (proposed_reserve_for_promo > budget_overrun) {
652 proposed_reserve_for_promo -= budget_overrun;
653 // Dead code:
654 // proposed_old_reserve -= budget_overrun;
655 } else {
656 budget_overrun -= proposed_reserve_for_promo;
657 proposed_reserve_for_promo = 0;
658 proposed_reserve_for_mixed = (proposed_reserve_for_mixed > budget_overrun)? proposed_reserve_for_mixed - budget_overrun: 0;
659 // Dead code:
660 // Note: proposed_reserve_for_promo is 0 and proposed_reserve_for_mixed may equal 0.
661 // proposed_old_reserve = proposed_reserve_for_mixed;
662 }
663 }
664
665 assert(old_region_deficit == 0 || old_region_surplus == 0,
666 "Only surplus (%zu) or deficit (%zu), never both", old_region_surplus, old_region_deficit);
667 assert(young_reserve + proposed_reserve_for_mixed + proposed_reserve_for_promo <= old_currently_available + young_available,
668 "Cannot reserve more memory than is available: %zu + %zu + %zu <= %zu + %zu",
669 young_reserve, proposed_reserve_for_mixed, proposed_reserve_for_promo, old_currently_available, young_available);
670
671 // deficit/surplus adjustments to generation sizes will precede rebuild
672 young_generation()->set_evacuation_reserve(young_reserve);
673 old_generation()->set_evacuation_reserve(proposed_reserve_for_mixed);
674 old_generation()->set_promoted_reserve(proposed_reserve_for_promo);
675 }
676
677 void ShenandoahGenerationalHeap::coalesce_and_fill_old_regions(bool concurrent) {
678 class ShenandoahGlobalCoalesceAndFill : public WorkerTask {
679 private:
680 ShenandoahPhaseTimings::Phase _phase;
681 ShenandoahRegionIterator _regions;
682 public:
683 explicit ShenandoahGlobalCoalesceAndFill(ShenandoahPhaseTimings::Phase phase) :
684 WorkerTask("Shenandoah Global Coalesce"),
685 _phase(phase) {}
686
687 void work(uint worker_id) override {
688 ShenandoahWorkerTimingsTracker timer(_phase,
689 ShenandoahPhaseTimings::Work,
690 worker_id, true);
691 ShenandoahHeapRegion* region;
692 while ((region = _regions.next()) != nullptr) {
693 // old region is not in the collection set and was not immediately trashed
694 if (region->is_old() && region->is_active() && !region->is_humongous()) {
695 // Reset the coalesce and fill boundary because this is a global collect
696 // and cannot be preempted by young collects. We want to be sure the entire
697 // region is coalesced here and does not resume from a previously interrupted
698 // or completed coalescing.
699 region->begin_preemptible_coalesce_and_fill();
700 region->oop_coalesce_and_fill(false);
701 }
702 }
703 }
704 };
705
706 ShenandoahPhaseTimings::Phase phase = concurrent ?
707 ShenandoahPhaseTimings::conc_coalesce_and_fill :
708 ShenandoahPhaseTimings::degen_gc_coalesce_and_fill;
709
710 // This is not cancellable
711 ShenandoahGlobalCoalesceAndFill coalesce(phase);
712 workers()->run_task(&coalesce);
713 old_generation()->set_parsable(true);
714 }
715
716 template<bool CONCURRENT>
717 class ShenandoahGenerationalUpdateHeapRefsTask : public WorkerTask {
718 private:
719 // For update refs, _generation will be young or global. Mixed collections use the young generation.
720 ShenandoahGeneration* _generation;
721 ShenandoahGenerationalHeap* _heap;
722 ShenandoahRegionIterator* _regions;
723 ShenandoahRegionChunkIterator* _work_chunks;
724
725 public:
726 ShenandoahGenerationalUpdateHeapRefsTask(ShenandoahGeneration* generation,
727 ShenandoahRegionIterator* regions,
728 ShenandoahRegionChunkIterator* work_chunks) :
729 WorkerTask("Shenandoah Update References"),
730 _generation(generation),
731 _heap(ShenandoahGenerationalHeap::heap()),
732 _regions(regions),
733 _work_chunks(work_chunks)
734 {
735 const bool old_bitmap_stable = _heap->old_generation()->is_mark_complete();
736 log_debug(gc, remset)("Update refs, scan remembered set using bitmap: %s", BOOL_TO_STR(old_bitmap_stable));
737 }
738
739 void work(uint worker_id) override {
740 if (CONCURRENT) {
741 ShenandoahConcurrentWorkerSession worker_session(worker_id);
742 SuspendibleThreadSetJoiner stsj;
743 do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
744 } else {
745 ShenandoahParallelWorkerSession worker_session(worker_id);
746 do_work<ShenandoahNonConcUpdateRefsClosure>(worker_id);
747 }
748 }
749
750 private:
751 template<class T>
752 void do_work(uint worker_id) {
753 T cl;
754
755 if (CONCURRENT && (worker_id == 0)) {
756 // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
757 // results of evacuation. These reserves are no longer necessary because evacuation has completed.
758 size_t cset_regions = _heap->collection_set()->count();
759
760 // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
761 // to the mutator free set. At the end of GC, we will have cset_regions newly evacuated fully empty regions from
762 // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
763 // next GC cycle.
764 _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
765 }
766 // If !CONCURRENT, there's no value in expanding Mutator free set
767
768 ShenandoahHeapRegion* r = _regions->next();
769 // We update references for global, mixed, and young collections.
770 assert(_generation->is_mark_complete(), "Expected complete marking");
771 ShenandoahMarkingContext* const ctx = _heap->marking_context();
772 bool is_mixed = _heap->collection_set()->has_old_regions();
773 while (r != nullptr) {
774 HeapWord* update_watermark = r->get_update_watermark();
775 assert(update_watermark >= r->bottom(), "sanity");
776
777 log_debug(gc)("Update refs worker " UINT32_FORMAT ", looking at region %zu", worker_id, r->index());
778 if (r->is_active() && !r->is_cset()) {
779 if (r->is_young()) {
780 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
781 } else if (r->is_old()) {
782 if (_generation->is_global()) {
783
784 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
785 }
786 // Otherwise, this is an old region in a young or mixed cycle. Process it during a second phase, below.
787 } else {
788 // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions
789 // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's
790 // active status may propagate at a different speed than the changing of the region's affiliation.
791
792 // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen
793 // by this thread before the region's affiliation() is seen by this thread.
794
795 // It's ok for this race to occur because the newly transformed region does not have any references to be
796 // updated.
797
798 assert(r->get_update_watermark() == r->bottom(),
799 "%s Region %zu is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE",
800 r->affiliation_name(), r->index());
801 }
802 }
803
804 if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
805 return;
806 }
807
808 r = _regions->next();
809 }
810
811 if (_generation->is_young()) {
812 // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered
813 // set processing if not in generational mode or if GLOBAL mode.
814
815 // After this thread has exhausted its traditional update-refs work, it continues with updating refs within
816 // remembered set. The remembered set workload is better balanced between threads, so threads that are "behind"
817 // can catch up with other threads during this phase, allowing all threads to work more effectively in parallel.
818 update_references_in_remembered_set(worker_id, cl, ctx, is_mixed);
819 }
820 }
821
822 template<class T>
823 void update_references_in_remembered_set(uint worker_id, T &cl, const ShenandoahMarkingContext* ctx, bool is_mixed) {
824
825 struct ShenandoahRegionChunk assignment;
826 ShenandoahScanRemembered* scanner = _heap->old_generation()->card_scan();
827
828 while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) {
829 // Keep grabbing next work chunk to process until finished, or asked to yield
830 ShenandoahHeapRegion* r = assignment._r;
831 if (r->is_active() && !r->is_cset() && r->is_old()) {
832 HeapWord* start_of_range = r->bottom() + assignment._chunk_offset;
833 HeapWord* end_of_range = r->get_update_watermark();
834 if (end_of_range > start_of_range + assignment._chunk_size) {
835 end_of_range = start_of_range + assignment._chunk_size;
836 }
837
838 if (start_of_range >= end_of_range) {
839 continue;
840 }
841
842 // Old region in a young cycle or mixed cycle.
843 if (is_mixed) {
844 if (r->is_humongous()) {
845 // Need to examine both dirty and clean cards during mixed evac.
846 r->oop_iterate_humongous_slice_all(&cl,start_of_range, assignment._chunk_size);
847 } else {
848 // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced
849 // and filled. This will use mark bits to find objects that need to be updated.
850 update_references_in_old_region(cl, ctx, scanner, r, start_of_range, end_of_range);
851 }
852 } else {
853 // This is a young evacuation
854 size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster;
855 size_t clusters = assignment._chunk_size / cluster_size;
856 assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries");
857 scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, worker_id);
858 }
859 }
860 }
861 }
862
863 template<class T>
864 void update_references_in_old_region(T &cl, const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner,
865 const ShenandoahHeapRegion* r, HeapWord* start_of_range,
866 HeapWord* end_of_range) const {
867 // In case last object in my range spans boundary of my chunk, I may need to scan all the way to top()
868 ShenandoahObjectToOopBoundedClosure<T> objs(&cl, start_of_range, r->top());
869
870 // Any object that begins in a previous range is part of a different scanning assignment. Any object that
871 // starts after end_of_range is also not my responsibility. (Either allocated during evacuation, so does
872 // not hold pointers to from-space, or is beyond the range of my assigned work chunk.)
873
874 // Find the first object that begins in my range, if there is one. Note that `p` will be set to `end_of_range`
875 // when no live object is found in the range.
876 HeapWord* tams = ctx->top_at_mark_start(r);
877 HeapWord* p = get_first_object_start_word(ctx, scanner, tams, start_of_range, end_of_range);
878
879 while (p < end_of_range) {
880 // p is known to point to the beginning of marked object obj
881 oop obj = cast_to_oop(p);
882 objs.do_object(obj);
883 HeapWord* prev_p = p;
884 p += obj->size();
885 if (p < tams) {
886 p = ctx->get_next_marked_addr(p, tams);
887 // If there are no more marked objects before tams, this returns tams. Note that tams is
888 // either >= end_of_range, or tams is the start of an object that is marked.
889 }
890 assert(p != prev_p, "Lack of forward progress");
891 }
892 }
893
894 HeapWord* get_first_object_start_word(const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner, HeapWord* tams,
895 HeapWord* start_of_range, HeapWord* end_of_range) const {
896 HeapWord* p = start_of_range;
897
898 if (p >= tams) {
899 // We cannot use ctx->is_marked(obj) to test whether an object begins at this address. Instead,
900 // we need to use the remembered set crossing map to advance p to the first object that starts
901 // within the enclosing card.
902 size_t card_index = scanner->card_index_for_addr(start_of_range);
903 while (true) {
904 HeapWord* first_object = scanner->first_object_in_card(card_index);
905 if (first_object != nullptr) {
906 p = first_object;
907 break;
908 } else if (scanner->addr_for_card_index(card_index + 1) < end_of_range) {
909 card_index++;
910 } else {
911 // Signal that no object was found in range
912 p = end_of_range;
913 break;
914 }
915 }
916 } else if (!ctx->is_marked(cast_to_oop(p))) {
917 p = ctx->get_next_marked_addr(p, tams);
918 // If there are no more marked objects before tams, this returns tams.
919 // Note that tams is either >= end_of_range, or tams is the start of an object that is marked.
920 }
921 return p;
922 }
923 };
924
925 void ShenandoahGenerationalHeap::update_heap_references(ShenandoahGeneration* generation, bool concurrent) {
926 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
927 const uint nworkers = workers()->active_workers();
928 ShenandoahRegionChunkIterator work_list(nworkers);
929 if (concurrent) {
930 ShenandoahGenerationalUpdateHeapRefsTask<true> task(generation, &_update_refs_iterator, &work_list);
931 workers()->run_task(&task);
932 } else {
933 ShenandoahGenerationalUpdateHeapRefsTask<false> task(generation, &_update_refs_iterator, &work_list);
934 workers()->run_task(&task);
935 }
936
937 if (ShenandoahEnableCardStats) {
938 // Only do this if we are collecting card stats
939 ShenandoahScanRemembered* card_scan = old_generation()->card_scan();
940 assert(card_scan != nullptr, "Card table must exist when card stats are enabled");
941 card_scan->log_card_stats(nworkers, CARD_STAT_UPDATE_REFS);
942 }
943 }
944
945 struct ShenandoahCompositeRegionClosure {
946 template<typename C1, typename C2>
947 class Closure : public ShenandoahHeapRegionClosure {
948 private:
949 C1 &_c1;
950 C2 &_c2;
951
952 public:
953 Closure(C1 &c1, C2 &c2) : ShenandoahHeapRegionClosure(), _c1(c1), _c2(c2) {}
954
955 void heap_region_do(ShenandoahHeapRegion* r) override {
956 _c1.heap_region_do(r);
957 _c2.heap_region_do(r);
958 }
959
960 bool is_thread_safe() override {
961 return _c1.is_thread_safe() && _c2.is_thread_safe();
962 }
963 };
964
965 template<typename C1, typename C2>
966 static Closure<C1, C2> of(C1 &c1, C2 &c2) {
967 return Closure<C1, C2>(c1, c2);
968 }
969 };
970
971 class ShenandoahUpdateRegionAges : public ShenandoahHeapRegionClosure {
972 private:
973 ShenandoahMarkingContext* _ctx;
974
975 public:
976 explicit ShenandoahUpdateRegionAges(ShenandoahMarkingContext* ctx) : _ctx(ctx) { }
977
978 void heap_region_do(ShenandoahHeapRegion* r) override {
979 // Maintenance of region age must follow evacuation in order to account for
980 // evacuation allocations within survivor regions. We consult region age during
981 // the subsequent evacuation to determine whether certain objects need to
982 // be promoted.
983 if (r->is_young() && r->is_active()) {
984 HeapWord *tams = _ctx->top_at_mark_start(r);
985 HeapWord *top = r->top();
986
987 // Allocations move the watermark when top moves. However, compacting
988 // objects will sometimes lower top beneath the watermark, after which,
989 // attempts to read the watermark will assert out (watermark should not be
990 // higher than top).
991 if (top > tams) {
992 // There have been allocations in this region since the start of the cycle.
993 // Any objects new to this region must not assimilate elevated age.
994 r->reset_age();
995 } else {
996 r->increment_age();
997 }
998 }
999 }
1000
1001 bool is_thread_safe() override {
1002 return true;
1003 }
1004 };
1005
1006 void ShenandoahGenerationalHeap::final_update_refs_update_region_states() {
1007 ShenandoahSynchronizePinnedRegionStates pins;
1008 ShenandoahUpdateRegionAges ages(marking_context());
1009 auto cl = ShenandoahCompositeRegionClosure::of(pins, ages);
1010 parallel_heap_region_iterate(&cl);
1011 }
1012
1013 void ShenandoahGenerationalHeap::complete_degenerated_cycle() {
1014 shenandoah_assert_heaplocked_or_safepoint();
1015 if (!old_generation()->is_parsable()) {
1016 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_coalesce_and_fill);
1017 coalesce_and_fill_old_regions(false);
1018 }
1019
1020 old_generation()->maybe_log_promotion_failure_stats(false);
1021 }
1022
1023 void ShenandoahGenerationalHeap::complete_concurrent_cycle() {
1024 if (!old_generation()->is_parsable()) {
1025 // Class unloading may render the card offsets unusable, so we must rebuild them before
1026 // the next remembered set scan. We _could_ let the control thread do this sometime after
1027 // the global cycle has completed and before the next young collection, but under memory
1028 // pressure the control thread may not have the time (that is, because it's running back
1029 // to back GCs). In that scenario, we would have to make the old regions parsable before
1030 // we could start a young collection. This could delay the start of the young cycle and
1031 // throw off the heuristics.
1032 entry_global_coalesce_and_fill();
1033 }
1034
1035 old_generation()->maybe_log_promotion_failure_stats(true);
1036 }
1037
1038 void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() {
1039 const char* msg = "Coalescing and filling old regions";
1040 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
1041
1042 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
1043 EventMark em("%s", msg);
1044 ShenandoahWorkerScope scope(workers(),
1045 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
1046 "concurrent coalesce and fill");
1047
1048 coalesce_and_fill_old_regions(true);
1049 }
1050
1051 void ShenandoahGenerationalHeap::update_region_ages(ShenandoahMarkingContext* ctx) {
1052 ShenandoahUpdateRegionAges cl(ctx);
1053 parallel_heap_region_iterate(&cl);
1054 }