1 /*
2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
3 * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shenandoah/shenandoahAgeCensus.hpp"
27 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahFreeSet.hpp"
30 #include "gc/shenandoah/shenandoahGeneration.hpp"
31 #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp"
32 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
33 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
37 #include "gc/shenandoah/shenandoahInitLogger.hpp"
38 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
39 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
40 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
41 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
42 #include "gc/shenandoah/shenandoahRegulatorThread.hpp"
43 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
44 #include "gc/shenandoah/shenandoahUtils.hpp"
45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
46 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
47 #include "logging/log.hpp"
48 #include "utilities/events.hpp"
49
50
51 class ShenandoahGenerationalInitLogger : public ShenandoahInitLogger {
52 public:
53 static void print() {
54 ShenandoahGenerationalInitLogger logger;
55 logger.print_all();
56 }
57 protected:
58 void print_gc_specific() override {
59 ShenandoahInitLogger::print_gc_specific();
60
61 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
62 log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name());
63 log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name());
64 }
65 };
66
67 size_t ShenandoahGenerationalHeap::calculate_min_plab() {
68 return align_up(PLAB::min_size(), CardTable::card_size_in_words());
69 }
70
71 size_t ShenandoahGenerationalHeap::calculate_max_plab() {
72 size_t MaxTLABSizeWords = ShenandoahHeapRegion::max_tlab_size_words();
73 return align_down(MaxTLABSizeWords, CardTable::card_size_in_words());
74 }
75
76 // Returns size in bytes
77 size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc() const {
78 return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available());
79 }
80
81 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
82 ShenandoahHeap(policy),
83 _age_census(nullptr),
84 _min_plab_size(calculate_min_plab()),
85 _max_plab_size(calculate_max_plab()),
86 _regulator_thread(nullptr),
87 _young_gen_memory_pool(nullptr),
88 _old_gen_memory_pool(nullptr) {
89 assert(is_aligned(_min_plab_size, CardTable::card_size_in_words()), "min_plab_size must be aligned");
90 assert(is_aligned(_max_plab_size, CardTable::card_size_in_words()), "max_plab_size must be aligned");
91 }
92
93 void ShenandoahGenerationalHeap::initialize_generations() {
94 ShenandoahHeap::initialize_generations();
95 _young_generation->post_initialize(this);
96 _old_generation->post_initialize(this);
97 }
98
99 void ShenandoahGenerationalHeap::post_initialize() {
100 ShenandoahHeap::post_initialize();
101 _age_census = new ShenandoahAgeCensus();
102 }
103
104 void ShenandoahGenerationalHeap::post_initialize_heuristics() {
105 ShenandoahHeap::post_initialize_heuristics();
106 _young_generation->post_initialize_heuristics();
107 _old_generation->post_initialize_heuristics();
108 }
109
110 void ShenandoahGenerationalHeap::print_init_logger() const {
111 ShenandoahGenerationalInitLogger logger;
112 logger.print_all();
113 }
114
115 void ShenandoahGenerationalHeap::initialize_heuristics() {
116 // Initialize global generation and heuristics even in generational mode.
117 ShenandoahHeap::initialize_heuristics();
118
119 _young_generation = new ShenandoahYoungGeneration(max_workers());
120 _old_generation = new ShenandoahOldGeneration(max_workers());
121 _young_generation->initialize_heuristics(mode());
122 _old_generation->initialize_heuristics(mode());
123 }
124
125 void ShenandoahGenerationalHeap::initialize_serviceability() {
126 assert(mode()->is_generational(), "Only for the generational mode");
127 _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this);
128 _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this);
129 cycle_memory_manager()->add_pool(_young_gen_memory_pool);
130 cycle_memory_manager()->add_pool(_old_gen_memory_pool);
131 stw_memory_manager()->add_pool(_young_gen_memory_pool);
132 stw_memory_manager()->add_pool(_old_gen_memory_pool);
133 }
134
135 GrowableArray<MemoryPool*> ShenandoahGenerationalHeap::memory_pools() {
136 assert(mode()->is_generational(), "Only for the generational mode");
137 GrowableArray<MemoryPool*> memory_pools(2);
138 memory_pools.append(_young_gen_memory_pool);
139 memory_pools.append(_old_gen_memory_pool);
140 return memory_pools;
141 }
142
143 void ShenandoahGenerationalHeap::initialize_controller() {
144 auto control_thread = new ShenandoahGenerationalControlThread();
145 _control_thread = control_thread;
146 _regulator_thread = new ShenandoahRegulatorThread(control_thread);
147 }
148
149 void ShenandoahGenerationalHeap::gc_threads_do(ThreadClosure* tcl) const {
150 if (!shenandoah_policy()->is_at_shutdown()) {
151 ShenandoahHeap::gc_threads_do(tcl);
152 tcl->do_thread(regulator_thread());
153 }
154 }
155
156 void ShenandoahGenerationalHeap::stop() {
157 ShenandoahHeap::stop();
158 regulator_thread()->stop();
159 }
160
161 void ShenandoahGenerationalHeap::start_idle_span() {
162 young_generation()->heuristics()->start_idle_span();
163 }
164
165 bool ShenandoahGenerationalHeap::requires_barriers(stackChunkOop obj) const {
166 if (is_idle()) {
167 return false;
168 }
169
170 if (is_concurrent_young_mark_in_progress() && is_in_young(obj) && !marking_context()->allocated_after_mark_start(obj)) {
171 // We are marking young, this object is in young, and it is below the TAMS
172 return true;
173 }
174
175 if (is_in_old(obj)) {
176 // Card marking barriers are required for objects in the old generation
177 return true;
178 }
179
180 if (has_forwarded_objects()) {
181 // Object may have pointers that need to be updated
182 return true;
183 }
184
185 return false;
186 }
187
188 void ShenandoahGenerationalHeap::evacuate_collection_set(ShenandoahGeneration* generation, bool concurrent) {
189 ShenandoahRegionIterator regions;
190 ShenandoahGenerationalEvacuationTask task(this, generation, ®ions, concurrent, false /* only promote regions */);
191 workers()->run_task(&task);
192 }
193
194 void ShenandoahGenerationalHeap::promote_regions_in_place(ShenandoahGeneration* generation, bool concurrent) {
195 ShenandoahRegionIterator regions;
196 ShenandoahGenerationalEvacuationTask task(this, generation, ®ions, concurrent, true /* only promote regions */);
197 workers()->run_task(&task);
198 }
199
200 oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) {
201 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
202 if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
203 // This thread went through the OOM during evac protocol and it is safe to return
204 // the forward pointer. It must not attempt to evacuate anymore.
205 return ShenandoahBarrierSet::resolve_forwarded(p);
206 }
207
208 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
209
210 ShenandoahHeapRegion* from_region = heap_region_containing(p);
211 assert(!from_region->is_humongous(), "never evacuate humongous objects");
212
213 // Try to keep the object in the same generation
214 const ShenandoahAffiliation target_gen = from_region->affiliation();
215
216 if (target_gen == YOUNG_GENERATION) {
217 markWord mark = p->mark();
218 if (mark.is_marked()) {
219 // Already forwarded.
220 return ShenandoahBarrierSet::resolve_forwarded(p);
221 }
222
223 if (mark.has_displaced_mark_helper()) {
224 // We don't want to deal with MT here just to ensure we read the right mark word.
225 // Skip the potential promotion attempt for this one.
226 } else if (age_census()->is_tenurable(from_region->age() + mark.age())) {
227 // If the object is tenurable, try to promote it
228 oop result = try_evacuate_object<YOUNG_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
229
230 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
231 if (result != nullptr) {
232 return result;
233 }
234 }
235 return try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(p, thread, from_region->age());
236 }
237
238 assert(target_gen == OLD_GENERATION, "Expected evacuation to old");
239 return try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(p, thread, from_region->age());
240 }
241
242 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
243 // to OLD_GENERATION.
244 template<ShenandoahAffiliation FROM_GENERATION, ShenandoahAffiliation TO_GENERATION>
245 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint from_region_age) {
246 bool alloc_from_lab = true;
247 bool has_plab = false;
248 HeapWord* copy = nullptr;
249
250 markWord mark = p->mark();
251 if (ShenandoahForwarding::is_forwarded(mark)) {
252 return ShenandoahForwarding::get_forwardee(p);
253 }
254 size_t old_size = ShenandoahForwarding::size(p);
255 size_t size = p->copy_size(old_size, mark);
256
257 constexpr bool is_promotion = (TO_GENERATION == OLD_GENERATION) && (FROM_GENERATION == YOUNG_GENERATION);
258
259 #ifdef ASSERT
260 if (ShenandoahOOMDuringEvacALot &&
261 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
262 copy = nullptr;
263 } else {
264 #endif
265 if (UseTLAB) {
266 switch (TO_GENERATION) {
267 case YOUNG_GENERATION: {
268 copy = allocate_from_gclab(thread, size);
269 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
270 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
271 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
272 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
273 copy = allocate_from_gclab(thread, size);
274 // If we still get nullptr, we'll try a shared allocation below.
275 }
276 break;
277 }
278 case OLD_GENERATION: {
279 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
280 if (plab != nullptr) {
281 has_plab = true;
282 copy = allocate_from_plab(thread, size, is_promotion);
283 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) &&
284 ShenandoahThreadLocalData::plab_retries_enabled(thread)) {
285 // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because
286 // the requested object does not fit within the current plab but the plab still has an "abundance" of memory,
287 // where abundance is defined as >= ShenGenHeap::plab_min_size(). In the former case, we try shrinking the
288 // desired PLAB size to the minimum and retry PLAB allocation to avoid cascading of shared memory allocations.
289 // Shrinking the desired PLAB size may allow us to eke out a small PLAB while staying beneath evacuation reserve.
290 if (plab->words_remaining() < plab_min_size()) {
291 ShenandoahThreadLocalData::set_plab_size(thread, plab_min_size());
292 copy = allocate_from_plab(thread, size, is_promotion);
293 // If we still get nullptr, we'll try a shared allocation below.
294 if (copy == nullptr) {
295 // If retry fails, don't continue to retry until we have success (probably in next GC pass)
296 ShenandoahThreadLocalData::disable_plab_retries(thread);
297 }
298 }
299 // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs.
300 }
301 }
302 break;
303 }
304 default: {
305 ShouldNotReachHere();
306 break;
307 }
308 }
309 }
310
311 if (copy == nullptr) {
312 // If we failed to allocate in LAB, we'll try a shared allocation.
313 if (!is_promotion || !has_plab || (size > PLAB::min_size())) {
314 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, TO_GENERATION, is_promotion);
315 copy = allocate_memory(req);
316 alloc_from_lab = false;
317 }
318 // else, we leave copy equal to nullptr, signaling a promotion failure below if appropriate.
319 // We choose not to promote objects smaller than size_threshold by way of shared allocations as this is too
320 // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future
321 // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= size_threshhold).
322 }
323 #ifdef ASSERT
324 }
325 #endif
326
327 if (copy == nullptr) {
328 if (TO_GENERATION == OLD_GENERATION) {
329 if (FROM_GENERATION == YOUNG_GENERATION) {
330 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
331 old_generation()->handle_failed_promotion(thread, size);
332 return nullptr;
333 } else {
334 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
335 // after the evacuation threads have finished.
336 old_generation()->handle_failed_evacuation();
337 }
338 }
339
340 control_thread()->handle_alloc_failure_evac(size);
341 oom_evac_handler()->handle_out_of_memory_during_evacuation();
342 return ShenandoahBarrierSet::resolve_forwarded(p);
343 }
344
345 if (ShenandoahEvacTracking) {
346 evac_tracker()->begin_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
347 }
348
349 // Copy the object:
350 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
351 oop copy_val = cast_to_oop(copy);
352
353 // Initialize the identity hash on the copy before installing the forwarding
354 // pointer, using the mark word we captured earlier. We must do this before
355 // the CAS so that the copy is fully initialized when it becomes visible to
356 // other threads. Using the captured mark (rather than re-reading the copy's
357 // mark) avoids races with other threads that may have evacuated p and
358 // installed a forwarding pointer in the meantime.
359 if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
360 copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
361 }
362
363 // Update the age of the evacuated object
364 if (TO_GENERATION == YOUNG_GENERATION && is_aging_cycle()) {
365 increase_object_age(copy_val, from_region_age + 1);
366 }
367
368 // Try to install the new forwarding pointer.
369 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
370 if (result == copy_val) {
371 // Successfully evacuated. Our copy is now the public one!
372
373 // This is necessary for virtual thread support. This uses the mark word without
374 // considering that it may now be a forwarding pointer (and could therefore crash).
375 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
376 // that lost the evacuation race (and will therefore not become visible). It is
377 // safe to do this on the public copy (this is also done during concurrent mark).
378 ContinuationGCSupport::relativize_stack_chunk(copy_val);
379
380 if (ShenandoahEvacTracking) {
381 // Record that the evacuation succeeded
382 evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
383 }
384
385 if (TO_GENERATION == OLD_GENERATION) {
386 old_generation()->handle_evacuation(copy, size);
387 }
388 } else {
389 // Failed to evacuate. We need to deal with the object that is left behind. Since this
390 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
391 // But if it happens to contain references to evacuated regions, those references would
392 // not get updated for this stale copy during this cycle, and we will crash while scanning
393 // it the next cycle.
394 if (alloc_from_lab) {
395 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
396 // object will overwrite this stale copy, or the filler object on LAB retirement will
397 // do this.
398 switch (TO_GENERATION) {
399 case YOUNG_GENERATION: {
400 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
401 break;
402 }
403 case OLD_GENERATION: {
404 ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size);
405 if (is_promotion) {
406 ShenandoahThreadLocalData::subtract_from_plab_promoted(thread, size * HeapWordSize);
407 }
408 break;
409 }
410 default: {
411 ShouldNotReachHere();
412 break;
413 }
414 }
415 } else {
416 // For non-LAB allocations, we have no way to retract the allocation, and
417 // have to explicitly overwrite the copy with the filler object. With that overwrite,
418 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
419 assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
420 fill_with_object(copy, size);
421 }
422 }
423 shenandoah_assert_correct(nullptr, result);
424 return result;
425 }
426
427 template oop ShenandoahGenerationalHeap::try_evacuate_object<YOUNG_GENERATION, YOUNG_GENERATION>(oop p, Thread* thread, uint from_region_age);
428 template oop ShenandoahGenerationalHeap::try_evacuate_object<YOUNG_GENERATION, OLD_GENERATION>(oop p, Thread* thread, uint from_region_age);
429 template oop ShenandoahGenerationalHeap::try_evacuate_object<OLD_GENERATION, OLD_GENERATION>(oop p, Thread* thread, uint from_region_age);
430
431 inline HeapWord* ShenandoahGenerationalHeap::allocate_from_plab(Thread* thread, size_t size, bool is_promotion) {
432 assert(UseTLAB, "TLABs should be enabled");
433
434 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
435 HeapWord* obj;
436
437 if (plab == nullptr) {
438 assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name());
439 // No PLABs in this thread, fallback to shared allocation
440 return nullptr;
441 } else if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
442 return nullptr;
443 }
444 // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy
445 obj = plab->allocate(size);
446 if ((obj == nullptr) && (plab->words_remaining() < plab_min_size())) {
447 // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations
448 obj = allocate_from_plab_slow(thread, size, is_promotion);
449 }
450 // if plab->words_remaining() >= ShenGenHeap::heap()->plab_min_size(), just return nullptr so we can use a shared allocation
451 if (obj == nullptr) {
452 return nullptr;
453 }
454
455 if (is_promotion) {
456 ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize);
457 }
458 return obj;
459 }
460
461 // Establish a new PLAB and allocate size HeapWords within it.
462 HeapWord* ShenandoahGenerationalHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) {
463 assert(mode()->is_generational(), "PLABs only relevant to generational GC");
464
465 const size_t plab_min_size = this->plab_min_size();
466 // PLABs are aligned to card boundaries to avoid synchronization with concurrent
467 // allocations in other PLABs.
468 const size_t min_size = (size > plab_min_size)? align_up(size, CardTable::card_size_in_words()): plab_min_size;
469
470 // Figure out size of new PLAB, using value determined at last refill.
471 size_t cur_size = ShenandoahThreadLocalData::plab_size(thread);
472 if (cur_size == 0) {
473 cur_size = plab_min_size;
474 }
475
476 // Expand aggressively, doubling at each refill in this epoch, ceiling at plab_max_size()
477 const size_t future_size = MIN2(cur_size * 2, plab_max_size());
478 // Doubling, starting at a card-multiple, should give us a card-multiple. (Ceiling and floor
479 // are card multiples.)
480 assert(is_aligned(future_size, CardTable::card_size_in_words()), "Card multiple by construction, future_size: %zu"
481 ", card_size: %u, cur_size: %zu, max: %zu",
482 future_size, CardTable::card_size_in_words(), cur_size, plab_max_size());
483
484 // Record new heuristic value even if we take any shortcut. This captures
485 // the case when moderately-sized objects always take a shortcut. At some point,
486 // heuristics should catch up with them. Note that the requested cur_size may
487 // not be honored, but we remember that this is the preferred size.
488 log_debug(gc, plab)("Set next PLAB refill size: %zu bytes", future_size * HeapWordSize);
489 ShenandoahThreadLocalData::set_plab_size(thread, future_size);
490
491 if (cur_size < size) {
492 // The PLAB to be allocated is still not large enough to hold the object. Fall back to shared allocation.
493 // This avoids retiring perfectly good PLABs in order to represent a single large object allocation.
494 log_debug(gc, plab)("Current PLAB size (%zu) is too small for %zu", cur_size * HeapWordSize, size * HeapWordSize);
495 return nullptr;
496 }
497
498 // Retire current PLAB, and allocate a new one.
499 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
500 if (plab->words_remaining() < plab_min_size) {
501 // Retire current PLAB. This takes care of any PLAB book-keeping.
502 // retire_plab() registers the remnant filler object with the remembered set scanner without a lock.
503 // Since PLABs are card-aligned, concurrent registrations in other PLABs don't interfere.
504 retire_plab(plab, thread);
505
506 size_t actual_size = 0;
507 HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size);
508 if (plab_buf == nullptr) {
509 if (min_size == plab_min_size) {
510 // Disable PLAB promotions for this thread because we cannot even allocate a minimal PLAB. This allows us
511 // to fail faster on subsequent promotion attempts.
512 ShenandoahThreadLocalData::disable_plab_promotions(thread);
513 }
514 return nullptr;
515 } else {
516 ShenandoahThreadLocalData::enable_plab_retries(thread);
517 }
518 // Since the allocated PLAB may have been down-sized for alignment, plab->allocate(size) below may still fail.
519 if (ZeroTLAB) {
520 // ... and clear it.
521 Copy::zero_to_words(plab_buf, actual_size);
522 } else {
523 // ...and zap just allocated object.
524 #ifdef ASSERT
525 // Skip mangling the space corresponding to the object header to
526 // ensure that the returned space is not considered parsable by
527 // any concurrent GC thread.
528 size_t hdr_size = oopDesc::header_size();
529 Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
530 #endif // ASSERT
531 }
532 assert(is_aligned(actual_size, CardTable::card_size_in_words()), "Align by design");
533 plab->set_buf(plab_buf, actual_size);
534 if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
535 return nullptr;
536 }
537 return plab->allocate(size);
538 } else {
539 // If there's still at least min_size() words available within the current plab, don't retire it. Let's nibble
540 // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request
541 // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we
542 // reduce the likelihood of evacuation failures, and we reduce the need for downsizing our PLABs.
543 return nullptr;
544 }
545 }
546
547 HeapWord* ShenandoahGenerationalHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) {
548 // Align requested sizes to card-sized multiples. Align down so that we don't violate max size of TLAB.
549 assert(is_aligned(min_size, CardTable::card_size_in_words()), "Align by design");
550 assert(word_size >= min_size, "Requested PLAB is too small");
551
552 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size);
553 // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread
554 // if we are at risk of infringing on the old-gen evacuation budget.
555 HeapWord* res = allocate_memory(req);
556 if (res != nullptr) {
557 *actual_size = req.actual_size();
558 } else {
559 *actual_size = 0;
560 }
561 assert(is_aligned(res, CardTable::card_size_in_words()), "Align by design");
562 return res;
563 }
564
565 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab, Thread* thread) {
566 // We don't enforce limits on plab evacuations. We let it consume all available old-gen memory in order to reduce
567 // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion
568 // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any
569 // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle.
570
571 // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to
572 // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions.
573 // 1. Some of the plab may have been dedicated to evacuations.
574 // 2. Some of the plab may have been abandoned due to waste (at the end of the plab).
575 size_t not_promoted =
576 ShenandoahThreadLocalData::get_plab_actual_size(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread);
577 ShenandoahThreadLocalData::reset_plab_promoted(thread);
578 ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
579 if (not_promoted > 0) {
580 log_debug(gc, plab)("Retire PLAB, unexpend unpromoted: %zu", not_promoted * HeapWordSize);
581 old_generation()->unexpend_promoted(not_promoted);
582 }
583 const size_t original_waste = plab->waste();
584 HeapWord* const top = plab->top();
585
586 // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable.
587 // It adds the size of this unused memory, in words, to plab->waste().
588 plab->retire();
589 if (top != nullptr && plab->waste() > original_waste && is_in_old(top)) {
590 // If retiring the plab created a filler object, then we need to register it with our card scanner so it can
591 // safely walk the region backing the plab.
592 log_debug(gc, plab)("retire_plab() is registering remnant of size %zu at " PTR_FORMAT,
593 (plab->waste() - original_waste) * HeapWordSize, p2i(top));
594 // No lock is necessary because the PLAB memory is aligned on card boundaries.
595 old_generation()->card_scan()->register_object_without_lock(top);
596 }
597 }
598
599 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) {
600 Thread* thread = Thread::current();
601 retire_plab(plab, thread);
602 }
603
604 // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations
605 // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to
606 // mutator_xfer_limit, and any surplus is transferred to the young generation. mutator_xfer_limit is
607 // the maximum we're able to transfer from young to old. This is called at the end of GC, as we prepare
608 // for the idle span that precedes the next GC.
609 void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t mutator_xfer_limit,
610 size_t old_trashed_regions, size_t young_trashed_regions) {
611 shenandoah_assert_heaplocked();
612 // We can limit the old reserve to the size of anticipated promotions:
613 // max_old_reserve is an upper bound on memory evacuated from old and promoted to old,
614 // clamped by the old generation space available.
615 //
616 // Here's the algebra.
617 // Let SOEP = ShenandoahOldEvacPercent,
618 // OE = old evac,
619 // YE = young evac, and
620 // TE = total evac = OE + YE
621 // By definition:
622 // SOEP/100 = OE/TE
623 // = OE/(OE+YE)
624 // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
625 // = OE/YE
626 // => OE = YE*SOEP/(100-SOEP)
627
628 // We have to be careful in the event that SOEP is set to 100 by the user.
629 assert(ShenandoahOldEvacPercent <= 100, "Error");
630 const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
631
632 ShenandoahOldGeneration* old_gen = old_generation();
633 size_t old_capacity = old_gen->max_capacity();
634 size_t old_usage = old_gen->used(); // includes humongous waste
635 size_t old_currently_available =
636 ((old_capacity >= old_usage)? old_capacity - old_usage: 0) + old_trashed_regions * region_size_bytes;
637
638 ShenandoahYoungGeneration* young_gen = young_generation();
639 size_t young_capacity = young_gen->max_capacity();
640 size_t young_usage = young_gen->used(); // includes humongous waste
641 size_t young_available = ((young_capacity >= young_usage)? young_capacity - young_usage: 0);
642 size_t freeset_available = free_set()->available_locked();
643 if (young_available > freeset_available) {
644 young_available = freeset_available;
645 }
646 young_available += young_trashed_regions * region_size_bytes;
647
648 // The free set will reserve this amount of memory to hold young evacuations (initialized to the ideal reserve)
649 size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100;
650
651 // If ShenandoahOldEvacPercent equals 100, max_old_reserve is limited only by mutator_xfer_limit and young_reserve
652 const size_t bound_on_old_reserve =
653 ((old_currently_available + mutator_xfer_limit + young_reserve) * ShenandoahOldEvacPercent) / 100;
654 size_t proposed_max_old = ((ShenandoahOldEvacPercent == 100)?
655 bound_on_old_reserve:
656 MIN2((young_reserve * ShenandoahOldEvacPercent) / (100 - ShenandoahOldEvacPercent),
657 bound_on_old_reserve));
658 if (young_reserve > young_available) {
659 young_reserve = young_available;
660 }
661
662 // Decide how much old space we should reserve for a mixed collection
663 size_t proposed_reserve_for_mixed = 0;
664 const size_t old_fragmented_available =
665 old_currently_available - (old_generation()->free_unaffiliated_regions() + old_trashed_regions) * region_size_bytes;
666
667 if (old_fragmented_available > proposed_max_old) {
668 // In this case, the old_fragmented_available is greater than the desired amount of evacuation to old.
669 // We'll use all of this memory to hold results of old evacuation, and we'll give back to the young generation
670 // any old regions that are not fragmented.
671 //
672 // This scenario may happen after we have promoted many regions in place, and each of these regions had non-zero
673 // unused memory, so there is now an abundance of old-fragmented available memory, even more than the desired
674 // percentage for old reserve. We cannot transfer these fragmented regions back to young. Instead we make the
675 // best of the situation by using this fragmented memory for both promotions and evacuations.
676
677 proposed_max_old = old_fragmented_available;
678 }
679 // Otherwise: old_fragmented_available <= proposed_max_old. Do not shrink proposed_max_old from the original computation.
680
681 // Though we initially set proposed_reserve_for_promo to equal the entirety of old fragmented available, we have the
682 // opportunity below to shift some of this memory into the proposed_reserve_for_mixed.
683 size_t proposed_reserve_for_promo = old_fragmented_available;
684 const size_t max_old_reserve = proposed_max_old;
685
686 const size_t mixed_candidate_live_memory = old_generation()->unprocessed_collection_candidates_live_memory();
687 const bool doing_mixed = (mixed_candidate_live_memory > 0);
688 if (doing_mixed) {
689 // In the ideal, all of the memory reserved for mixed evacuation would be unfragmented, but we don't enforce
690 // this. Note that the initial value of max_evac_need is conservative because we may not evacuate all of the
691 // remaining mixed evacuation candidates in a single cycle.
692 const size_t max_evac_need = (size_t) (mixed_candidate_live_memory * ShenandoahOldEvacWaste);
693 assert(old_currently_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes,
694 "Unaffiliated available must be less than total available");
695
696 // We prefer to evacuate all of mixed into unfragmented memory, and will expand old in order to do so, unless
697 // we already have too much fragmented available memory in old.
698 proposed_reserve_for_mixed = max_evac_need;
699 if (proposed_reserve_for_mixed + proposed_reserve_for_promo > max_old_reserve) {
700 // We're trying to reserve more memory than is available. So we need to shrink our reserves.
701 size_t excess_reserves = (proposed_reserve_for_mixed + proposed_reserve_for_promo) - max_old_reserve;
702 // We need to shrink reserves by excess_reserves. We prefer to shrink by reducing promotion, giving priority to mixed
703 // evacuation. If the promotion reserve is larger than the amount we need to shrink by, do all the shrinkage there.
704 if (proposed_reserve_for_promo > excess_reserves) {
705 proposed_reserve_for_promo -= excess_reserves;
706 } else {
707 // Otherwise, we'll shrink promotion reserve to zero and we'll shrink the mixed-evac reserve by the remaining excess.
708 excess_reserves -= proposed_reserve_for_promo;
709 proposed_reserve_for_promo = 0;
710 proposed_reserve_for_mixed -= excess_reserves;
711 }
712 }
713 }
714 assert(proposed_reserve_for_mixed + proposed_reserve_for_promo <= max_old_reserve,
715 "Reserve for mixed (%zu) plus reserve for promotions (%zu) must be less than maximum old reserve (%zu)",
716 proposed_reserve_for_mixed, proposed_reserve_for_promo, max_old_reserve);
717
718 // Decide how much additional space we should reserve for promotions from young. We give priority to mixed evacations
719 // over promotions.
720 const size_t promo_load = old_generation()->get_promotion_potential();
721 const bool doing_promotions = promo_load > 0;
722
723 // promo_load represents the combined total of live memory within regions that have reached tenure age. The true
724 // promotion potential is larger than this, because individual objects within regions that have not yet reached tenure
725 // age may be promotable. On the other hand, some of the objects that we intend to promote in the next GC cycle may
726 // die before they are next marked. In the future, the promo_load will include the total size of tenurable objects
727 // residing in regions that have not yet reached tenure age.
728
729 if (doing_promotions) {
730 // We are always doing promotions, even when old_generation->get_promotion_potential() returns 0. As currently implemented,
731 // get_promotion_potential() only knows the total live memory contained within young-generation regions whose age is
732 // tenurable. It does not know whether that memory will still be live at the end of the next mark cycle, and it doesn't
733 // know how much memory is contained within objects whose individual ages are tenurable, which reside in regions with
734 // non-tenurable age. We use this, as adjusted by ShenandoahPromoEvacWaste, as an approximation of the total amount of
735 // memory to be promoted. In the near future, we expect to implement a change that will allow get_promotion_potential()
736 // to account also for the total memory contained within individual objects that are tenure-ready even when they do
737 // not reside in aged regions. This will represent a conservative over approximation of promotable memory because
738 // some of these objects may die before the next GC cycle executes.
739
740 // Be careful not to ask for too much promotion reserves. We have observed jtreg test failures under which a greedy
741 // promotion reserve causes a humongous allocation which is awaiting a full GC to fail (specifically
742 // gc/TestAllocHumongousFragment.java). This happens if too much of the memory reclaimed by the full GC
743 // is immediately reserved so that it cannot be allocated by the waiting mutator. It's not clear that this
744 // particular test is representative of the needs of typical GenShen users. It is really a test of high frequency
745 // Full GCs under heap fragmentation stress.
746
747 size_t promo_need = (size_t) (promo_load * ShenandoahPromoEvacWaste);
748 if (promo_need > proposed_reserve_for_promo) {
749 const size_t available_for_additional_promotions =
750 max_old_reserve - (proposed_reserve_for_mixed + proposed_reserve_for_promo);
751 if (proposed_reserve_for_promo + available_for_additional_promotions >= promo_need) {
752 proposed_reserve_for_promo = promo_need;
753 } else {
754 proposed_reserve_for_promo += available_for_additional_promotions;
755 }
756 }
757 }
758 // else, leave proposed_reserve_for_promo as is. By default, it is initialized to represent old_fragmented_available.
759
760 // This is the total old we want to reserve (initialized to the ideal reserve)
761 size_t proposed_old_reserve = proposed_reserve_for_mixed + proposed_reserve_for_promo;
762
763 // We now check if the old generation is running a surplus or a deficit.
764 size_t old_region_deficit = 0;
765 size_t old_region_surplus = 0;
766
767 size_t mutator_region_xfer_limit = mutator_xfer_limit / region_size_bytes;
768 // align the mutator_xfer_limit on region size
769 mutator_xfer_limit = mutator_region_xfer_limit * region_size_bytes;
770
771 if (old_currently_available >= proposed_old_reserve) {
772 // We are running a surplus, so the old region surplus can go to young
773 const size_t old_surplus = old_currently_available - proposed_old_reserve;
774 old_region_surplus = old_surplus / region_size_bytes;
775 const size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_trashed_regions;
776 old_region_surplus = MIN2(old_region_surplus, unaffiliated_old_regions);
777 old_generation()->set_region_balance(checked_cast<ssize_t>(old_region_surplus));
778 old_currently_available -= old_region_surplus * region_size_bytes;
779 young_available += old_region_surplus * region_size_bytes;
780 } else if (old_currently_available + mutator_xfer_limit >= proposed_old_reserve) {
781 // We know that old_currently_available < proposed_old_reserve because above test failed. Expand old_currently_available.
782 // Mutator's xfer limit is sufficient to satisfy our need: transfer all memory from there.
783 size_t old_deficit = proposed_old_reserve - old_currently_available;
784 old_region_deficit = (old_deficit + region_size_bytes - 1) / region_size_bytes;
785 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit));
786 old_currently_available += old_region_deficit * region_size_bytes;
787 young_available -= old_region_deficit * region_size_bytes;
788 } else {
789 // We know that (old_currently_available < proposed_old_reserve) and
790 // (old_currently_available + mutator_xfer_limit < proposed_old_reserve) because above tests failed.
791 // We need to shrink proposed_old_reserves.
792
793 // We could potentially shrink young_reserves in order to further expand proposed_old_reserves. Let's not bother. The
794 // important thing is that we keep a total amount of memory in reserve in preparation for the next GC cycle. At
795 // the time we choose the next collection set, we'll have an opportunity to shift some of these young reserves
796 // into old reserves if that makes sense.
797
798 // Start by taking all of mutator_xfer_limit into old_currently_available.
799 size_t old_region_deficit = mutator_region_xfer_limit;
800 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit));
801 old_currently_available += old_region_deficit * region_size_bytes;
802 young_available -= old_region_deficit * region_size_bytes;
803
804 assert(old_currently_available < proposed_old_reserve,
805 "Old currently available (%zu) must be less than old reserve (%zu)", old_currently_available, proposed_old_reserve);
806
807 // There's not enough memory to satisfy our desire. Scale back our old-gen intentions. We prefer to satisfy
808 // the budget_overrun entirely from the promotion reserve, if that is large enough. Otherwise, we'll satisfy
809 // the overrun from a combination of promotion and mixed-evacuation reserves.
810 size_t budget_overrun = proposed_old_reserve - old_currently_available;
811 if (proposed_reserve_for_promo > budget_overrun) {
812 proposed_reserve_for_promo -= budget_overrun;
813 // Dead code:
814 // proposed_old_reserve -= budget_overrun;
815 } else {
816 budget_overrun -= proposed_reserve_for_promo;
817 proposed_reserve_for_promo = 0;
818 proposed_reserve_for_mixed = (proposed_reserve_for_mixed > budget_overrun)? proposed_reserve_for_mixed - budget_overrun: 0;
819 // Dead code:
820 // Note: proposed_reserve_for_promo is 0 and proposed_reserve_for_mixed may equal 0.
821 // proposed_old_reserve = proposed_reserve_for_mixed;
822 }
823 }
824
825 assert(old_region_deficit == 0 || old_region_surplus == 0,
826 "Only surplus (%zu) or deficit (%zu), never both", old_region_surplus, old_region_deficit);
827 assert(young_reserve + proposed_reserve_for_mixed + proposed_reserve_for_promo <= old_currently_available + young_available,
828 "Cannot reserve more memory than is available: %zu + %zu + %zu <= %zu + %zu",
829 young_reserve, proposed_reserve_for_mixed, proposed_reserve_for_promo, old_currently_available, young_available);
830
831 // deficit/surplus adjustments to generation sizes will precede rebuild
832 young_generation()->set_evacuation_reserve(young_reserve);
833 old_generation()->set_evacuation_reserve(proposed_reserve_for_mixed);
834 old_generation()->set_promoted_reserve(proposed_reserve_for_promo);
835 }
836
837 void ShenandoahGenerationalHeap::coalesce_and_fill_old_regions(bool concurrent) {
838 class ShenandoahGlobalCoalesceAndFill : public WorkerTask {
839 private:
840 ShenandoahPhaseTimings::Phase _phase;
841 ShenandoahRegionIterator _regions;
842 public:
843 explicit ShenandoahGlobalCoalesceAndFill(ShenandoahPhaseTimings::Phase phase) :
844 WorkerTask("Shenandoah Global Coalesce"),
845 _phase(phase) {}
846
847 void work(uint worker_id) override {
848 ShenandoahWorkerTimingsTracker timer(_phase,
849 ShenandoahPhaseTimings::ScanClusters,
850 worker_id, true);
851 ShenandoahHeapRegion* region;
852 while ((region = _regions.next()) != nullptr) {
853 // old region is not in the collection set and was not immediately trashed
854 if (region->is_old() && region->is_active() && !region->is_humongous()) {
855 // Reset the coalesce and fill boundary because this is a global collect
856 // and cannot be preempted by young collects. We want to be sure the entire
857 // region is coalesced here and does not resume from a previously interrupted
858 // or completed coalescing.
859 region->begin_preemptible_coalesce_and_fill();
860 region->oop_coalesce_and_fill(false);
861 }
862 }
863 }
864 };
865
866 ShenandoahPhaseTimings::Phase phase = concurrent ?
867 ShenandoahPhaseTimings::conc_coalesce_and_fill :
868 ShenandoahPhaseTimings::degen_gc_coalesce_and_fill;
869
870 // This is not cancellable
871 ShenandoahGlobalCoalesceAndFill coalesce(phase);
872 workers()->run_task(&coalesce);
873 old_generation()->set_parsable(true);
874 }
875
876 template<bool CONCURRENT>
877 class ShenandoahGenerationalUpdateHeapRefsTask : public WorkerTask {
878 private:
879 // For update refs, _generation will be young or global. Mixed collections use the young generation.
880 ShenandoahGeneration* _generation;
881 ShenandoahGenerationalHeap* _heap;
882 ShenandoahRegionIterator* _regions;
883 ShenandoahRegionChunkIterator* _work_chunks;
884
885 public:
886 ShenandoahGenerationalUpdateHeapRefsTask(ShenandoahGeneration* generation,
887 ShenandoahRegionIterator* regions,
888 ShenandoahRegionChunkIterator* work_chunks) :
889 WorkerTask("Shenandoah Update References"),
890 _generation(generation),
891 _heap(ShenandoahGenerationalHeap::heap()),
892 _regions(regions),
893 _work_chunks(work_chunks)
894 {
895 const bool old_bitmap_stable = _heap->old_generation()->is_mark_complete();
896 log_debug(gc, remset)("Update refs, scan remembered set using bitmap: %s", BOOL_TO_STR(old_bitmap_stable));
897 }
898
899 void work(uint worker_id) override {
900 if (CONCURRENT) {
901 ShenandoahConcurrentWorkerSession worker_session(worker_id);
902 ShenandoahSuspendibleThreadSetJoiner stsj;
903 do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
904 } else {
905 ShenandoahParallelWorkerSession worker_session(worker_id);
906 do_work<ShenandoahNonConcUpdateRefsClosure>(worker_id);
907 }
908 }
909
910 private:
911 template<class T>
912 void do_work(uint worker_id) {
913 T cl;
914
915 if (CONCURRENT && (worker_id == 0)) {
916 // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
917 // results of evacuation. These reserves are no longer necessary because evacuation has completed.
918 size_t cset_regions = _heap->collection_set()->count();
919
920 // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
921 // to the mutator free set. At the end of GC, we will have cset_regions newly evacuated fully empty regions from
922 // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
923 // next GC cycle.
924 _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
925 }
926 // If !CONCURRENT, there's no value in expanding Mutator free set
927
928 ShenandoahHeapRegion* r = _regions->next();
929 // We update references for global, mixed, and young collections.
930 assert(_generation->is_mark_complete(), "Expected complete marking");
931 ShenandoahMarkingContext* const ctx = _heap->marking_context();
932 bool is_mixed = _heap->collection_set()->has_old_regions();
933 while (r != nullptr) {
934 HeapWord* update_watermark = r->get_update_watermark();
935 assert(update_watermark >= r->bottom(), "sanity");
936
937 log_debug(gc)("Update refs worker " UINT32_FORMAT ", looking at region %zu", worker_id, r->index());
938 if (r->is_active() && !r->is_cset()) {
939 if (r->is_young()) {
940 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
941 } else if (r->is_old()) {
942 if (_generation->is_global()) {
943
944 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
945 }
946 // Otherwise, this is an old region in a young or mixed cycle. Process it during a second phase, below.
947 } else {
948 // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions
949 // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's
950 // active status may propagate at a different speed than the changing of the region's affiliation.
951
952 // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen
953 // by this thread before the region's affiliation() is seen by this thread.
954
955 // It's ok for this race to occur because the newly transformed region does not have any references to be
956 // updated.
957
958 assert(r->get_update_watermark() == r->bottom(),
959 "%s Region %zu is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE",
960 r->affiliation_name(), r->index());
961 }
962 }
963
964 if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
965 return;
966 }
967
968 r = _regions->next();
969 }
970
971 if (_generation->is_young()) {
972 // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered
973 // set processing if not in generational mode or if GLOBAL mode.
974
975 // After this thread has exhausted its traditional update-refs work, it continues with updating refs within
976 // remembered set. The remembered set workload is better balanced between threads, so threads that are "behind"
977 // can catch up with other threads during this phase, allowing all threads to work more effectively in parallel.
978 update_references_in_remembered_set(worker_id, cl, ctx, is_mixed);
979 }
980 }
981
982 template<class T>
983 void update_references_in_remembered_set(uint worker_id, T &cl, const ShenandoahMarkingContext* ctx, bool is_mixed) {
984
985 struct ShenandoahRegionChunk assignment;
986 ShenandoahScanRemembered* scanner = _heap->old_generation()->card_scan();
987
988 while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) {
989 // Keep grabbing next work chunk to process until finished, or asked to yield
990 ShenandoahHeapRegion* r = assignment._r;
991 if (r->is_active() && !r->is_cset() && r->is_old()) {
992 HeapWord* start_of_range = r->bottom() + assignment._chunk_offset;
993 HeapWord* end_of_range = r->get_update_watermark();
994 if (end_of_range > start_of_range + assignment._chunk_size) {
995 end_of_range = start_of_range + assignment._chunk_size;
996 }
997
998 if (start_of_range >= end_of_range) {
999 continue;
1000 }
1001
1002 // Old region in a young cycle or mixed cycle.
1003 if (is_mixed) {
1004 if (r->is_humongous()) {
1005 // Need to examine both dirty and clean cards during mixed evac.
1006 r->oop_iterate_humongous_slice_all(&cl,start_of_range, assignment._chunk_size);
1007 } else {
1008 // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced
1009 // and filled. This will use mark bits to find objects that need to be updated.
1010 update_references_in_old_region(cl, ctx, scanner, r, start_of_range, end_of_range);
1011 }
1012 } else {
1013 // This is a young evacuation
1014 size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster;
1015 size_t clusters = assignment._chunk_size / cluster_size;
1016 assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries");
1017 scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, worker_id);
1018 }
1019 }
1020 }
1021 }
1022
1023 template<class T>
1024 void update_references_in_old_region(T &cl, const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner,
1025 const ShenandoahHeapRegion* r, HeapWord* start_of_range,
1026 HeapWord* end_of_range) const {
1027 // In case last object in my range spans boundary of my chunk, I may need to scan all the way to top()
1028 ShenandoahObjectToOopBoundedClosure<T> objs(&cl, start_of_range, r->top());
1029
1030 // Any object that begins in a previous range is part of a different scanning assignment. Any object that
1031 // starts after end_of_range is also not my responsibility. (Either allocated during evacuation, so does
1032 // not hold pointers to from-space, or is beyond the range of my assigned work chunk.)
1033
1034 // Find the first object that begins in my range, if there is one. Note that `p` will be set to `end_of_range`
1035 // when no live object is found in the range.
1036 HeapWord* tams = ctx->top_at_mark_start(r);
1037 HeapWord* p = get_first_object_start_word(ctx, scanner, tams, start_of_range, end_of_range);
1038
1039 while (p < end_of_range) {
1040 // p is known to point to the beginning of marked object obj
1041 oop obj = cast_to_oop(p);
1042 objs.do_object(obj);
1043 HeapWord* prev_p = p;
1044 p += obj->size();
1045 if (p < tams) {
1046 p = ctx->get_next_marked_addr(p, tams);
1047 // If there are no more marked objects before tams, this returns tams. Note that tams is
1048 // either >= end_of_range, or tams is the start of an object that is marked.
1049 }
1050 assert(p != prev_p, "Lack of forward progress");
1051 }
1052 }
1053
1054 HeapWord* get_first_object_start_word(const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner, HeapWord* tams,
1055 HeapWord* start_of_range, HeapWord* end_of_range) const {
1056 HeapWord* p = start_of_range;
1057
1058 if (p >= tams) {
1059 // We cannot use ctx->is_marked(obj) to test whether an object begins at this address. Instead,
1060 // we need to use the remembered set crossing map to advance p to the first object that starts
1061 // within the enclosing card.
1062 size_t card_index = scanner->card_index_for_addr(start_of_range);
1063 while (true) {
1064 HeapWord* first_object = scanner->first_object_in_card(card_index);
1065 if (first_object != nullptr) {
1066 p = first_object;
1067 break;
1068 } else if (scanner->addr_for_card_index(card_index + 1) < end_of_range) {
1069 card_index++;
1070 } else {
1071 // Signal that no object was found in range
1072 p = end_of_range;
1073 break;
1074 }
1075 }
1076 } else if (!ctx->is_marked(cast_to_oop(p))) {
1077 p = ctx->get_next_marked_addr(p, tams);
1078 // If there are no more marked objects before tams, this returns tams.
1079 // Note that tams is either >= end_of_range, or tams is the start of an object that is marked.
1080 }
1081 return p;
1082 }
1083 };
1084
1085 void ShenandoahGenerationalHeap::update_heap_references(ShenandoahGeneration* generation, bool concurrent) {
1086 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1087 const uint nworkers = workers()->active_workers();
1088 ShenandoahRegionChunkIterator work_list(nworkers);
1089 if (concurrent) {
1090 ShenandoahGenerationalUpdateHeapRefsTask<true> task(generation, &_update_refs_iterator, &work_list);
1091 workers()->run_task(&task);
1092 } else {
1093 ShenandoahGenerationalUpdateHeapRefsTask<false> task(generation, &_update_refs_iterator, &work_list);
1094 workers()->run_task(&task);
1095 }
1096
1097 if (ShenandoahEnableCardStats) {
1098 // Only do this if we are collecting card stats
1099 ShenandoahScanRemembered* card_scan = old_generation()->card_scan();
1100 assert(card_scan != nullptr, "Card table must exist when card stats are enabled");
1101 card_scan->log_card_stats(nworkers, CARD_STAT_UPDATE_REFS);
1102 }
1103 }
1104
1105 struct ShenandoahCompositeRegionClosure {
1106 template<typename C1, typename C2>
1107 class Closure : public ShenandoahHeapRegionClosure {
1108 private:
1109 C1 &_c1;
1110 C2 &_c2;
1111
1112 public:
1113 Closure(C1 &c1, C2 &c2) : ShenandoahHeapRegionClosure(), _c1(c1), _c2(c2) {}
1114
1115 void heap_region_do(ShenandoahHeapRegion* r) override {
1116 _c1.heap_region_do(r);
1117 _c2.heap_region_do(r);
1118 }
1119
1120 bool is_thread_safe() override {
1121 return _c1.is_thread_safe() && _c2.is_thread_safe();
1122 }
1123 };
1124
1125 template<typename C1, typename C2>
1126 static Closure<C1, C2> of(C1 &c1, C2 &c2) {
1127 return Closure<C1, C2>(c1, c2);
1128 }
1129 };
1130
1131 class ShenandoahUpdateRegionAges : public ShenandoahHeapRegionClosure {
1132 private:
1133 ShenandoahMarkingContext* _ctx;
1134
1135 public:
1136 explicit ShenandoahUpdateRegionAges(ShenandoahMarkingContext* ctx) : _ctx(ctx) { }
1137
1138 void heap_region_do(ShenandoahHeapRegion* r) override {
1139 // Maintenance of region age must follow evacuation in order to account for
1140 // evacuation allocations within survivor regions. We consult region age during
1141 // the subsequent evacuation to determine whether certain objects need to
1142 // be promoted.
1143 if (r->is_young() && r->is_active()) {
1144 HeapWord *tams = _ctx->top_at_mark_start(r);
1145 HeapWord *top = r->top();
1146
1147 // Allocations move the watermark when top moves. However, compacting
1148 // objects will sometimes lower top beneath the watermark, after which,
1149 // attempts to read the watermark will assert out (watermark should not be
1150 // higher than top).
1151 if (top > tams) {
1152 // There have been allocations in this region since the start of the cycle.
1153 // Any objects new to this region must not assimilate elevated age.
1154 r->reset_age();
1155 } else if (ShenandoahGenerationalHeap::heap()->is_aging_cycle()) {
1156 r->increment_age();
1157 }
1158 }
1159 }
1160
1161 bool is_thread_safe() override {
1162 return true;
1163 }
1164 };
1165
1166 void ShenandoahGenerationalHeap::final_update_refs_update_region_states() {
1167 ShenandoahSynchronizePinnedRegionStates pins;
1168 ShenandoahUpdateRegionAges ages(marking_context());
1169 auto cl = ShenandoahCompositeRegionClosure::of(pins, ages);
1170 parallel_heap_region_iterate(&cl);
1171 }
1172
1173 void ShenandoahGenerationalHeap::complete_degenerated_cycle() {
1174 shenandoah_assert_heaplocked_or_safepoint();
1175 if (!old_generation()->is_parsable()) {
1176 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_coalesce_and_fill);
1177 coalesce_and_fill_old_regions(false);
1178 }
1179
1180 log_info(gc, cset)("Degenerated cycle complete, promotions reserved: %zu, promotions expended: %zu, failed count: %zu, failed bytes: %zu",
1181 old_generation()->get_promoted_reserve(), old_generation()->get_promoted_expended(),
1182 old_generation()->get_promotion_failed_count(), old_generation()->get_promotion_failed_words() * HeapWordSize);
1183 }
1184
1185 void ShenandoahGenerationalHeap::complete_concurrent_cycle() {
1186 if (!old_generation()->is_parsable()) {
1187 // Class unloading may render the card offsets unusable, so we must rebuild them before
1188 // the next remembered set scan. We _could_ let the control thread do this sometime after
1189 // the global cycle has completed and before the next young collection, but under memory
1190 // pressure the control thread may not have the time (that is, because it's running back
1191 // to back GCs). In that scenario, we would have to make the old regions parsable before
1192 // we could start a young collection. This could delay the start of the young cycle and
1193 // throw off the heuristics.
1194 entry_global_coalesce_and_fill();
1195 }
1196
1197 log_info(gc, cset)("Concurrent cycle complete, promotions reserved: %zu, promotions expended: %zu, failed count: %zu, failed bytes: %zu",
1198 old_generation()->get_promoted_reserve(), old_generation()->get_promoted_expended(),
1199 old_generation()->get_promotion_failed_count(), old_generation()->get_promotion_failed_words() * HeapWordSize);
1200 }
1201
1202 void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() {
1203 const char* msg = "Coalescing and filling old regions";
1204 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
1205
1206 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
1207 EventMark em("%s", msg);
1208 ShenandoahWorkerScope scope(workers(),
1209 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
1210 "concurrent coalesce and fill");
1211
1212 coalesce_and_fill_old_regions(true);
1213 }
1214
1215 void ShenandoahGenerationalHeap::update_region_ages(ShenandoahMarkingContext* ctx) {
1216 ShenandoahUpdateRegionAges cl(ctx);
1217 parallel_heap_region_iterate(&cl);
1218 }