1 /*
2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shenandoah/shenandoahAgeCensus.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahFreeSet.hpp"
30 #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp"
31 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
32 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahInitLogger.hpp"
36 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
37 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
38 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
41 #include "gc/shenandoah/shenandoahRegulatorThread.hpp"
42 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
43 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
44 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
45 #include "gc/shenandoah/shenandoahUtils.hpp"
46 #include "logging/log.hpp"
47 #include "utilities/events.hpp"
48
49
50 class ShenandoahGenerationalInitLogger : public ShenandoahInitLogger {
51 public:
52 static void print() {
53 ShenandoahGenerationalInitLogger logger;
54 logger.print_all();
55 }
56
57 void print_heap() override {
58 ShenandoahInitLogger::print_heap();
59
60 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
61
62 ShenandoahYoungGeneration* young = heap->young_generation();
63 log_info(gc, init)("Young Generation Soft Size: " EXACTFMT, EXACTFMTARGS(young->soft_max_capacity()));
64 log_info(gc, init)("Young Generation Max: " EXACTFMT, EXACTFMTARGS(young->max_capacity()));
65
66 ShenandoahOldGeneration* old = heap->old_generation();
67 log_info(gc, init)("Old Generation Soft Size: " EXACTFMT, EXACTFMTARGS(old->soft_max_capacity()));
68 log_info(gc, init)("Old Generation Max: " EXACTFMT, EXACTFMTARGS(old->max_capacity()));
69 }
70
71 protected:
72 void print_gc_specific() override {
73 ShenandoahInitLogger::print_gc_specific();
74
75 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
76 log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name());
77 log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name());
78 }
79 };
80
81 size_t ShenandoahGenerationalHeap::calculate_min_plab() {
82 return align_up(PLAB::min_size(), CardTable::card_size_in_words());
83 }
84
85 size_t ShenandoahGenerationalHeap::calculate_max_plab() {
86 size_t MaxTLABSizeWords = ShenandoahHeapRegion::max_tlab_size_words();
87 return align_down(MaxTLABSizeWords, CardTable::card_size_in_words());
88 }
89
90 // Returns size in bytes
91 size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const {
92 return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available());
93 }
94
95 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
96 ShenandoahHeap(policy),
97 _age_census(nullptr),
98 _evac_tracker(new ShenandoahEvacuationTracker()),
99 _min_plab_size(calculate_min_plab()),
100 _max_plab_size(calculate_max_plab()),
101 _regulator_thread(nullptr),
102 _young_gen_memory_pool(nullptr),
103 _old_gen_memory_pool(nullptr) {
104 assert(is_aligned(_min_plab_size, CardTable::card_size_in_words()), "min_plab_size must be aligned");
105 assert(is_aligned(_max_plab_size, CardTable::card_size_in_words()), "max_plab_size must be aligned");
106 }
107
108 void ShenandoahGenerationalHeap::post_initialize() {
109 ShenandoahHeap::post_initialize();
110 _age_census = new ShenandoahAgeCensus();
111 }
112
113 void ShenandoahGenerationalHeap::print_init_logger() const {
114 ShenandoahGenerationalInitLogger logger;
115 logger.print_all();
116 }
117
118 void ShenandoahGenerationalHeap::print_tracing_info() const {
119 ShenandoahHeap::print_tracing_info();
120
121 LogTarget(Info, gc, stats) lt;
122 if (lt.is_enabled()) {
123 LogStream ls(lt);
124 ls.cr();
125 ls.cr();
126 evac_tracker()->print_global_on(&ls);
127 }
128 }
129
130 void ShenandoahGenerationalHeap::initialize_heuristics() {
131 // Initialize global generation and heuristics even in generational mode.
132 ShenandoahHeap::initialize_heuristics();
133
134 // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity
135 // for old would be total heap - minimum capacity of young. This means the sum of the maximum
136 // allowed for old and young could exceed the total heap size. It remains the case that the
137 // _actual_ capacity of young + old = total.
138 _generation_sizer.heap_size_changed(max_capacity());
139 size_t initial_capacity_young = _generation_sizer.max_young_size();
140 size_t max_capacity_young = _generation_sizer.max_young_size();
141 size_t initial_capacity_old = max_capacity() - max_capacity_young;
142 size_t max_capacity_old = max_capacity() - initial_capacity_young;
143
144 _young_generation = new ShenandoahYoungGeneration(max_workers(), max_capacity_young, initial_capacity_young);
145 _old_generation = new ShenandoahOldGeneration(max_workers(), max_capacity_old, initial_capacity_old);
146 _young_generation->initialize_heuristics(mode());
147 _old_generation->initialize_heuristics(mode());
148 }
149
150 void ShenandoahGenerationalHeap::initialize_serviceability() {
151 assert(mode()->is_generational(), "Only for the generational mode");
152 _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this);
153 _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this);
154 cycle_memory_manager()->add_pool(_young_gen_memory_pool);
155 cycle_memory_manager()->add_pool(_old_gen_memory_pool);
156 stw_memory_manager()->add_pool(_young_gen_memory_pool);
157 stw_memory_manager()->add_pool(_old_gen_memory_pool);
158 }
159
160 GrowableArray<MemoryPool*> ShenandoahGenerationalHeap::memory_pools() {
161 assert(mode()->is_generational(), "Only for the generational mode");
162 GrowableArray<MemoryPool*> memory_pools(2);
163 memory_pools.append(_young_gen_memory_pool);
164 memory_pools.append(_old_gen_memory_pool);
165 return memory_pools;
166 }
167
168 void ShenandoahGenerationalHeap::initialize_controller() {
169 auto control_thread = new ShenandoahGenerationalControlThread();
170 _control_thread = control_thread;
171 _regulator_thread = new ShenandoahRegulatorThread(control_thread);
172 }
173
174 void ShenandoahGenerationalHeap::gc_threads_do(ThreadClosure* tcl) const {
175 if (!shenandoah_policy()->is_at_shutdown()) {
176 ShenandoahHeap::gc_threads_do(tcl);
177 tcl->do_thread(regulator_thread());
178 }
179 }
180
181 void ShenandoahGenerationalHeap::stop() {
182 regulator_thread()->stop();
183 ShenandoahHeap::stop();
184 }
185
186 void ShenandoahGenerationalHeap::evacuate_collection_set(bool concurrent) {
187 ShenandoahRegionIterator regions;
188 ShenandoahGenerationalEvacuationTask task(this, ®ions, concurrent, false /* only promote regions */);
189 workers()->run_task(&task);
190 }
191
192 void ShenandoahGenerationalHeap::promote_regions_in_place(bool concurrent) {
193 ShenandoahRegionIterator regions;
194 ShenandoahGenerationalEvacuationTask task(this, ®ions, concurrent, true /* only promote regions */);
195 workers()->run_task(&task);
196 }
197
198 oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) {
199 assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
200 if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
201 // This thread went through the OOM during evac protocol and it is safe to return
202 // the forward pointer. It must not attempt to evacuate anymore.
203 return ShenandoahBarrierSet::resolve_forwarded(p);
204 }
205
206 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
207
208 ShenandoahHeapRegion* r = heap_region_containing(p);
209 assert(!r->is_humongous(), "never evacuate humongous objects");
210
211 ShenandoahAffiliation target_gen = r->affiliation();
212 // gc_generation() can change asynchronously and should not be used here.
213 assert(active_generation() != nullptr, "Error");
214 if (active_generation()->is_young() && target_gen == YOUNG_GENERATION) {
215 markWord mark = p->mark();
216 if (mark.is_marked()) {
217 // Already forwarded.
218 return ShenandoahBarrierSet::resolve_forwarded(p);
219 }
220
221 if (mark.has_displaced_mark_helper()) {
222 // We don't want to deal with MT here just to ensure we read the right mark word.
223 // Skip the potential promotion attempt for this one.
224 } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
225 oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
226 if (result != nullptr) {
227 return result;
228 }
229 // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
230 }
231 }
232 return try_evacuate_object(p, thread, r, target_gen);
233 }
234
235 // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
236 // to OLD_GENERATION.
237 oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
238 ShenandoahAffiliation target_gen) {
239 bool alloc_from_lab = true;
240 bool has_plab = false;
241 HeapWord* copy = nullptr;
242 size_t size = p->size();
243 bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
244
245 #ifdef ASSERT
246 if (ShenandoahOOMDuringEvacALot &&
247 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
248 copy = nullptr;
249 } else {
250 #endif
251 if (UseTLAB) {
252 switch (target_gen) {
253 case YOUNG_GENERATION: {
254 copy = allocate_from_gclab(thread, size);
255 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
256 // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
257 // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
258 ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
259 copy = allocate_from_gclab(thread, size);
260 // If we still get nullptr, we'll try a shared allocation below.
261 }
262 break;
263 }
264 case OLD_GENERATION: {
265 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
266 if (plab != nullptr) {
267 has_plab = true;
268 }
269 copy = allocate_from_plab(thread, size, is_promotion);
270 if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) &&
271 ShenandoahThreadLocalData::plab_retries_enabled(thread)) {
272 // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because
273 // the requested object does not fit within the current plab but the plab still has an "abundance" of memory,
274 // where abundance is defined as >= ShenGenHeap::plab_min_size(). In the former case, we try shrinking the
275 // desired PLAB size to the minimum and retry PLAB allocation to avoid cascading of shared memory allocations.
276 if (plab->words_remaining() < plab_min_size()) {
277 ShenandoahThreadLocalData::set_plab_size(thread, plab_min_size());
278 copy = allocate_from_plab(thread, size, is_promotion);
279 // If we still get nullptr, we'll try a shared allocation below.
280 if (copy == nullptr) {
281 // If retry fails, don't continue to retry until we have success (probably in next GC pass)
282 ShenandoahThreadLocalData::disable_plab_retries(thread);
283 }
284 }
285 // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs.
286 }
287 break;
288 }
289 default: {
290 ShouldNotReachHere();
291 break;
292 }
293 }
294 }
295
296 if (copy == nullptr) {
297 // If we failed to allocate in LAB, we'll try a shared allocation.
298 if (!is_promotion || !has_plab || (size > PLAB::min_size())) {
299 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen, is_promotion);
300 copy = allocate_memory(req);
301 alloc_from_lab = false;
302 }
303 // else, we leave copy equal to nullptr, signaling a promotion failure below if appropriate.
304 // We choose not to promote objects smaller than PLAB::min_size() by way of shared allocations, as this is too
305 // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future
306 // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= PLAB::min_size())
307 }
308 #ifdef ASSERT
309 }
310 #endif
311
312 if (copy == nullptr) {
313 if (target_gen == OLD_GENERATION) {
314 if (from_region->is_young()) {
315 // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
316 old_generation()->handle_failed_promotion(thread, size);
317 return nullptr;
318 } else {
319 // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
320 // after the evacuation threads have finished.
321 old_generation()->handle_failed_evacuation();
322 }
323 }
324
325 control_thread()->handle_alloc_failure_evac(size);
326
327 oom_evac_handler()->handle_out_of_memory_during_evacuation();
328
329 return ShenandoahBarrierSet::resolve_forwarded(p);
330 }
331
332 // Copy the object:
333 NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
334 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
335 oop copy_val = cast_to_oop(copy);
336
337 // Update the age of the evacuated object
338 if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
339 ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
340 }
341
342 // Try to install the new forwarding pointer.
343 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
344 if (result == copy_val) {
345 // Successfully evacuated. Our copy is now the public one!
346
347 // This is necessary for virtual thread support. This uses the mark word without
348 // considering that it may now be a forwarding pointer (and could therefore crash).
349 // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
350 // that lost the evacuation race (and will therefore not become visible). It is
351 // safe to do this on the public copy (this is also done during concurrent mark).
352 ContinuationGCSupport::relativize_stack_chunk(copy_val);
353
354 // Record that the evacuation succeeded
355 NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
356
357 if (target_gen == OLD_GENERATION) {
358 old_generation()->handle_evacuation(copy, size, from_region->is_young());
359 } else {
360 // When copying to the old generation above, we don't care
361 // about recording object age in the census stats.
362 assert(target_gen == YOUNG_GENERATION, "Error");
363 // We record this census only when simulating pre-adaptive tenuring behavior, or
364 // when we have been asked to record the census at evacuation rather than at mark
365 if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
366 evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
367 }
368 }
369 shenandoah_assert_correct(nullptr, copy_val);
370 return copy_val;
371 } else {
372 // Failed to evacuate. We need to deal with the object that is left behind. Since this
373 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
374 // But if it happens to contain references to evacuated regions, those references would
375 // not get updated for this stale copy during this cycle, and we will crash while scanning
376 // it the next cycle.
377 if (alloc_from_lab) {
378 // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
379 // object will overwrite this stale copy, or the filler object on LAB retirement will
380 // do this.
381 switch (target_gen) {
382 case YOUNG_GENERATION: {
383 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
384 break;
385 }
386 case OLD_GENERATION: {
387 ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size);
388 if (is_promotion) {
389 ShenandoahThreadLocalData::subtract_from_plab_promoted(thread, size * HeapWordSize);
390 }
391 break;
392 }
393 default: {
394 ShouldNotReachHere();
395 break;
396 }
397 }
398 } else {
399 // For non-LAB allocations, we have no way to retract the allocation, and
400 // have to explicitly overwrite the copy with the filler object. With that overwrite,
401 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
402 assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
403 fill_with_object(copy, size);
404 shenandoah_assert_correct(nullptr, copy_val);
405 // For non-LAB allocations, the object has already been registered
406 }
407 shenandoah_assert_correct(nullptr, result);
408 return result;
409 }
410 }
411
412 inline HeapWord* ShenandoahGenerationalHeap::allocate_from_plab(Thread* thread, size_t size, bool is_promotion) {
413 assert(UseTLAB, "TLABs should be enabled");
414
415 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
416 HeapWord* obj;
417
418 if (plab == nullptr) {
419 assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name());
420 // No PLABs in this thread, fallback to shared allocation
421 return nullptr;
422 } else if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
423 return nullptr;
424 }
425 // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy
426 obj = plab->allocate(size);
427 if ((obj == nullptr) && (plab->words_remaining() < plab_min_size())) {
428 // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations
429 obj = allocate_from_plab_slow(thread, size, is_promotion);
430 }
431 // if plab->words_remaining() >= ShenGenHeap::heap()->plab_min_size(), just return nullptr so we can use a shared allocation
432 if (obj == nullptr) {
433 return nullptr;
434 }
435
436 if (is_promotion) {
437 ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize);
438 }
439 return obj;
440 }
441
442 // Establish a new PLAB and allocate size HeapWords within it.
443 HeapWord* ShenandoahGenerationalHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) {
444 // New object should fit the PLAB size
445
446 assert(mode()->is_generational(), "PLABs only relevant to generational GC");
447 const size_t plab_min_size = this->plab_min_size();
448 // PLABs are aligned to card boundaries to avoid synchronization with concurrent
449 // allocations in other PLABs.
450 const size_t min_size = (size > plab_min_size)? align_up(size, CardTable::card_size_in_words()): plab_min_size;
451
452 // Figure out size of new PLAB, using value determined at last refill.
453 size_t cur_size = ShenandoahThreadLocalData::plab_size(thread);
454 if (cur_size == 0) {
455 cur_size = plab_min_size;
456 }
457
458 // Expand aggressively, doubling at each refill in this epoch, ceiling at plab_max_size()
459 size_t future_size = MIN2(cur_size * 2, plab_max_size());
460 // Doubling, starting at a card-multiple, should give us a card-multiple. (Ceiling and floor
461 // are card multiples.)
462 assert(is_aligned(future_size, CardTable::card_size_in_words()), "Card multiple by construction, future_size: " SIZE_FORMAT
463 ", card_size: " SIZE_FORMAT ", cur_size: " SIZE_FORMAT ", max: " SIZE_FORMAT,
464 future_size, (size_t) CardTable::card_size_in_words(), cur_size, plab_max_size());
465
466 // Record new heuristic value even if we take any shortcut. This captures
467 // the case when moderately-sized objects always take a shortcut. At some point,
468 // heuristics should catch up with them. Note that the requested cur_size may
469 // not be honored, but we remember that this is the preferred size.
470 log_debug(gc, free)("Set new PLAB size: " SIZE_FORMAT, future_size);
471 ShenandoahThreadLocalData::set_plab_size(thread, future_size);
472 if (cur_size < size) {
473 // The PLAB to be allocated is still not large enough to hold the object. Fall back to shared allocation.
474 // This avoids retiring perfectly good PLABs in order to represent a single large object allocation.
475 log_debug(gc, free)("Current PLAB size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, cur_size, size);
476 return nullptr;
477 }
478
479 // Retire current PLAB, and allocate a new one.
480 PLAB* plab = ShenandoahThreadLocalData::plab(thread);
481 if (plab->words_remaining() < plab_min_size) {
482 // Retire current PLAB. This takes care of any PLAB book-keeping.
483 // retire_plab() registers the remnant filler object with the remembered set scanner without a lock.
484 // Since PLABs are card-aligned, concurrent registrations in other PLABs don't interfere.
485 retire_plab(plab, thread);
486
487 size_t actual_size = 0;
488 HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size);
489 if (plab_buf == nullptr) {
490 if (min_size == plab_min_size) {
491 // Disable PLAB promotions for this thread because we cannot even allocate a minimal PLAB. This allows us
492 // to fail faster on subsequent promotion attempts.
493 ShenandoahThreadLocalData::disable_plab_promotions(thread);
494 }
495 return nullptr;
496 } else {
497 ShenandoahThreadLocalData::enable_plab_retries(thread);
498 }
499 // Since the allocated PLAB may have been down-sized for alignment, plab->allocate(size) below may still fail.
500 if (ZeroTLAB) {
501 // ... and clear it.
502 Copy::zero_to_words(plab_buf, actual_size);
503 } else {
504 // ...and zap just allocated object.
505 #ifdef ASSERT
506 // Skip mangling the space corresponding to the object header to
507 // ensure that the returned space is not considered parsable by
508 // any concurrent GC thread.
509 size_t hdr_size = oopDesc::header_size();
510 Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
511 #endif // ASSERT
512 }
513 assert(is_aligned(actual_size, CardTable::card_size_in_words()), "Align by design");
514 plab->set_buf(plab_buf, actual_size);
515 if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
516 return nullptr;
517 }
518 return plab->allocate(size);
519 } else {
520 // If there's still at least min_size() words available within the current plab, don't retire it. Let's nibble
521 // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request
522 // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we
523 // reduce the likelihood of evacuation failures, and we reduce the need for downsizing our PLABs.
524 return nullptr;
525 }
526 }
527
528 HeapWord* ShenandoahGenerationalHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) {
529 // Align requested sizes to card-sized multiples. Align down so that we don't violate max size of TLAB.
530 assert(is_aligned(min_size, CardTable::card_size_in_words()), "Align by design");
531 assert(word_size >= min_size, "Requested PLAB is too small");
532
533 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size);
534 // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread
535 // if we are at risk of infringing on the old-gen evacuation budget.
536 HeapWord* res = allocate_memory(req);
537 if (res != nullptr) {
538 *actual_size = req.actual_size();
539 } else {
540 *actual_size = 0;
541 }
542 assert(is_aligned(res, CardTable::card_size_in_words()), "Align by design");
543 return res;
544 }
545
546 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab, Thread* thread) {
547 // We don't enforce limits on plab evacuations. We let it consume all available old-gen memory in order to reduce
548 // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion
549 // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any
550 // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle.
551
552 // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to
553 // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions.
554 // 1. Some of the plab may have been dedicated to evacuations.
555 // 2. Some of the plab may have been abandoned due to waste (at the end of the plab).
556 size_t not_promoted =
557 ShenandoahThreadLocalData::get_plab_actual_size(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread);
558 ShenandoahThreadLocalData::reset_plab_promoted(thread);
559 ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
560 if (not_promoted > 0) {
561 old_generation()->unexpend_promoted(not_promoted);
562 }
563 const size_t original_waste = plab->waste();
564 HeapWord* const top = plab->top();
565
566 // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable.
567 // It adds the size of this unused memory, in words, to plab->waste().
568 plab->retire();
569 if (top != nullptr && plab->waste() > original_waste && is_in_old(top)) {
570 // If retiring the plab created a filler object, then we need to register it with our card scanner so it can
571 // safely walk the region backing the plab.
572 log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT,
573 plab->waste() - original_waste, p2i(top));
574 // No lock is necessary because the PLAB memory is aligned on card boundaries.
575 old_generation()->card_scan()->register_object_without_lock(top);
576 }
577 }
578
579 void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) {
580 Thread* thread = Thread::current();
581 retire_plab(plab, thread);
582 }
583
584 ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalHeap::balance_generations() {
585 shenandoah_assert_heaplocked_or_safepoint();
586
587 ShenandoahOldGeneration* old_gen = old_generation();
588 const ssize_t old_region_balance = old_gen->get_region_balance();
589 old_gen->set_region_balance(0);
590
591 if (old_region_balance > 0) {
592 const auto old_region_surplus = checked_cast<size_t>(old_region_balance);
593 const bool success = generation_sizer()->transfer_to_young(old_region_surplus);
594 return TransferResult {
595 success, old_region_surplus, "young"
596 };
597 }
598
599 if (old_region_balance < 0) {
600 const auto old_region_deficit = checked_cast<size_t>(-old_region_balance);
601 const bool success = generation_sizer()->transfer_to_old(old_region_deficit);
602 if (!success) {
603 old_gen->handle_failed_transfer();
604 }
605 return TransferResult {
606 success, old_region_deficit, "old"
607 };
608 }
609
610 return TransferResult {true, 0, "none"};
611 }
612
613 // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations
614 // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to
615 // xfer_limit, and any surplus is transferred to the young generation.
616 // xfer_limit is the maximum we're able to transfer from young to old.
617 void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions) {
618
619 // We can limit the old reserve to the size of anticipated promotions:
620 // max_old_reserve is an upper bound on memory evacuated from old and promoted to old,
621 // clamped by the old generation space available.
622 //
623 // Here's the algebra.
624 // Let SOEP = ShenandoahOldEvacRatioPercent,
625 // OE = old evac,
626 // YE = young evac, and
627 // TE = total evac = OE + YE
628 // By definition:
629 // SOEP/100 = OE/TE
630 // = OE/(OE+YE)
631 // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
632 // = OE/YE
633 // => OE = YE*SOEP/(100-SOEP)
634
635 // We have to be careful in the event that SOEP is set to 100 by the user.
636 assert(ShenandoahOldEvacRatioPercent <= 100, "Error");
637 const size_t old_available = old_generation()->available();
638 // The free set will reserve this amount of memory to hold young evacuations
639 const size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100;
640
641 // In the case that ShenandoahOldEvacRatioPercent equals 100, max_old_reserve is limited only by xfer_limit.
642
643 const size_t bound_on_old_reserve = old_available + old_xfer_limit + young_reserve;
644 const size_t max_old_reserve = (ShenandoahOldEvacRatioPercent == 100)?
645 bound_on_old_reserve: MIN2((young_reserve * ShenandoahOldEvacRatioPercent) / (100 - ShenandoahOldEvacRatioPercent),
646 bound_on_old_reserve);
647
648 const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
649
650 // Decide how much old space we should reserve for a mixed collection
651 size_t reserve_for_mixed = 0;
652 if (old_generation()->has_unprocessed_collection_candidates()) {
653 // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we
654 // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation.
655 const size_t max_evac_need = (size_t)
656 (old_generation()->unprocessed_collection_candidates_live_memory() * ShenandoahOldEvacWaste);
657 assert(old_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes,
658 "Unaffiliated available must be less than total available");
659 const size_t old_fragmented_available =
660 old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes;
661 reserve_for_mixed = max_evac_need + old_fragmented_available;
662 if (reserve_for_mixed > max_old_reserve) {
663 reserve_for_mixed = max_old_reserve;
664 }
665 }
666
667 // Decide how much space we should reserve for promotions from young
668 size_t reserve_for_promo = 0;
669 const size_t promo_load = old_generation()->get_promotion_potential();
670 const bool doing_promotions = promo_load > 0;
671 if (doing_promotions) {
672 // We're promoting and have a bound on the maximum amount that can be promoted
673 assert(max_old_reserve >= reserve_for_mixed, "Sanity");
674 const size_t available_for_promotions = max_old_reserve - reserve_for_mixed;
675 reserve_for_promo = MIN2((size_t)(promo_load * ShenandoahPromoEvacWaste), available_for_promotions);
676 }
677
678 // This is the total old we want to ideally reserve
679 const size_t old_reserve = reserve_for_mixed + reserve_for_promo;
680 assert(old_reserve <= max_old_reserve, "cannot reserve more than max for old evacuations");
681
682 // We now check if the old generation is running a surplus or a deficit.
683 const size_t max_old_available = old_generation()->available() + old_cset_regions * region_size_bytes;
684 if (max_old_available >= old_reserve) {
685 // We are running a surplus, so the old region surplus can go to young
686 const size_t old_surplus = (max_old_available - old_reserve) / region_size_bytes;
687 const size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_cset_regions;
688 const size_t old_region_surplus = MIN2(old_surplus, unaffiliated_old_regions);
689 old_generation()->set_region_balance(checked_cast<ssize_t>(old_region_surplus));
690 } else {
691 // We are running a deficit which we'd like to fill from young.
692 // Ignore that this will directly impact young_generation()->max_capacity(),
693 // indirectly impacting young_reserve and old_reserve. These computations are conservative.
694 // Note that deficit is rounded up by one region.
695 const size_t old_need = (old_reserve - max_old_available + region_size_bytes - 1) / region_size_bytes;
696 const size_t max_old_region_xfer = old_xfer_limit / region_size_bytes;
697
698 // Round down the regions we can transfer from young to old. If we're running short
699 // on young-gen memory, we restrict the xfer. Old-gen collection activities will be
700 // curtailed if the budget is restricted.
701 const size_t old_region_deficit = MIN2(old_need, max_old_region_xfer);
702 old_generation()->set_region_balance(0 - checked_cast<ssize_t>(old_region_deficit));
703 }
704 }
705
706 void ShenandoahGenerationalHeap::reset_generation_reserves() {
707 young_generation()->set_evacuation_reserve(0);
708 old_generation()->set_evacuation_reserve(0);
709 old_generation()->set_promoted_reserve(0);
710 }
711
712 void ShenandoahGenerationalHeap::TransferResult::print_on(const char* when, outputStream* ss) const {
713 auto heap = ShenandoahGenerationalHeap::heap();
714 ShenandoahYoungGeneration* const young_gen = heap->young_generation();
715 ShenandoahOldGeneration* const old_gen = heap->old_generation();
716 const size_t young_available = young_gen->available();
717 const size_t old_available = old_gen->available();
718 ss->print_cr("After %s, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: "
719 PROPERFMT ", young_available: " PROPERFMT,
720 when,
721 success? "successfully transferred": "failed to transfer", region_count, region_destination,
722 PROPERFMTARGS(old_available), PROPERFMTARGS(young_available));
723 }
724
725 void ShenandoahGenerationalHeap::coalesce_and_fill_old_regions(bool concurrent) {
726 class ShenandoahGlobalCoalesceAndFill : public WorkerTask {
727 private:
728 ShenandoahPhaseTimings::Phase _phase;
729 ShenandoahRegionIterator _regions;
730 public:
731 explicit ShenandoahGlobalCoalesceAndFill(ShenandoahPhaseTimings::Phase phase) :
732 WorkerTask("Shenandoah Global Coalesce"),
733 _phase(phase) {}
734
735 void work(uint worker_id) override {
736 ShenandoahWorkerTimingsTracker timer(_phase,
737 ShenandoahPhaseTimings::ScanClusters,
738 worker_id, true);
739 ShenandoahHeapRegion* region;
740 while ((region = _regions.next()) != nullptr) {
741 // old region is not in the collection set and was not immediately trashed
742 if (region->is_old() && region->is_active() && !region->is_humongous()) {
743 // Reset the coalesce and fill boundary because this is a global collect
744 // and cannot be preempted by young collects. We want to be sure the entire
745 // region is coalesced here and does not resume from a previously interrupted
746 // or completed coalescing.
747 region->begin_preemptible_coalesce_and_fill();
748 region->oop_coalesce_and_fill(false);
749 }
750 }
751 }
752 };
753
754 ShenandoahPhaseTimings::Phase phase = concurrent ?
755 ShenandoahPhaseTimings::conc_coalesce_and_fill :
756 ShenandoahPhaseTimings::degen_gc_coalesce_and_fill;
757
758 // This is not cancellable
759 ShenandoahGlobalCoalesceAndFill coalesce(phase);
760 workers()->run_task(&coalesce);
761 old_generation()->set_parsable(true);
762 }
763
764 template<bool CONCURRENT>
765 class ShenandoahGenerationalUpdateHeapRefsTask : public WorkerTask {
766 private:
767 ShenandoahGenerationalHeap* _heap;
768 ShenandoahRegionIterator* _regions;
769 ShenandoahRegionChunkIterator* _work_chunks;
770
771 public:
772 explicit ShenandoahGenerationalUpdateHeapRefsTask(ShenandoahRegionIterator* regions,
773 ShenandoahRegionChunkIterator* work_chunks) :
774 WorkerTask("Shenandoah Update References"),
775 _heap(ShenandoahGenerationalHeap::heap()),
776 _regions(regions),
777 _work_chunks(work_chunks)
778 {
779 bool old_bitmap_stable = _heap->old_generation()->is_mark_complete();
780 log_debug(gc, remset)("Update refs, scan remembered set using bitmap: %s", BOOL_TO_STR(old_bitmap_stable));
781 }
782
783 void work(uint worker_id) {
784 if (CONCURRENT) {
785 ShenandoahConcurrentWorkerSession worker_session(worker_id);
786 ShenandoahSuspendibleThreadSetJoiner stsj;
787 do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
788 } else {
789 ShenandoahParallelWorkerSession worker_session(worker_id);
790 do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
791 }
792 }
793
794 private:
795 template<class T>
796 void do_work(uint worker_id) {
797 T cl;
798
799 if (CONCURRENT && (worker_id == 0)) {
800 // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
801 // results of evacuation. These reserves are no longer necessary because evacuation has completed.
802 size_t cset_regions = _heap->collection_set()->count();
803
804 // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
805 // to the mutator free set. At the end of GC, we will have cset_regions newly evacuated fully empty regions from
806 // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
807 // next GC cycle.
808 _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
809 }
810 // If !CONCURRENT, there's no value in expanding Mutator free set
811
812 ShenandoahHeapRegion* r = _regions->next();
813 // We update references for global, old, and young collections.
814 ShenandoahGeneration* const gc_generation = _heap->gc_generation();
815 shenandoah_assert_generations_reconciled();
816 assert(gc_generation->is_mark_complete(), "Expected complete marking");
817 ShenandoahMarkingContext* const ctx = _heap->marking_context();
818 bool is_mixed = _heap->collection_set()->has_old_regions();
819 while (r != nullptr) {
820 HeapWord* update_watermark = r->get_update_watermark();
821 assert(update_watermark >= r->bottom(), "sanity");
822
823 log_debug(gc)("Update refs worker " UINT32_FORMAT ", looking at region " SIZE_FORMAT, worker_id, r->index());
824 bool region_progress = false;
825 if (r->is_active() && !r->is_cset()) {
826 if (r->is_young()) {
827 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
828 region_progress = true;
829 } else if (r->is_old()) {
830 if (gc_generation->is_global()) {
831
832 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
833 region_progress = true;
834 }
835 // Otherwise, this is an old region in a young or mixed cycle. Process it during a second phase, below.
836 // Don't bother to report pacing progress in this case.
837 } else {
838 // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions
839 // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's
840 // active status may propagate at a different speed than the changing of the region's affiliation.
841
842 // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen
843 // by this thread before the region's affiliation() is seen by this thread.
844
845 // It's ok for this race to occur because the newly transformed region does not have any references to be
846 // updated.
847
848 assert(r->get_update_watermark() == r->bottom(),
849 "%s Region " SIZE_FORMAT " is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE",
850 r->affiliation_name(), r->index());
851 }
852 }
853
854 if (region_progress && ShenandoahPacing) {
855 _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
856 }
857
858 if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
859 return;
860 }
861
862 r = _regions->next();
863 }
864
865 if (!gc_generation->is_global()) {
866 // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered
867 // set processing if not in generational mode or if GLOBAL mode.
868
869 // After this thread has exhausted its traditional update-refs work, it continues with updating refs within
870 // remembered set. The remembered set workload is better balanced between threads, so threads that are "behind"
871 // can catch up with other threads during this phase, allowing all threads to work more effectively in parallel.
872 update_references_in_remembered_set(worker_id, cl, ctx, is_mixed);
873 }
874 }
875
876 template<class T>
877 void update_references_in_remembered_set(uint worker_id, T &cl, const ShenandoahMarkingContext* ctx, bool is_mixed) {
878
879 struct ShenandoahRegionChunk assignment;
880 ShenandoahScanRemembered* scanner = _heap->old_generation()->card_scan();
881
882 while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) {
883 // Keep grabbing next work chunk to process until finished, or asked to yield
884 ShenandoahHeapRegion* r = assignment._r;
885 if (r->is_active() && !r->is_cset() && r->is_old()) {
886 HeapWord* start_of_range = r->bottom() + assignment._chunk_offset;
887 HeapWord* end_of_range = r->get_update_watermark();
888 if (end_of_range > start_of_range + assignment._chunk_size) {
889 end_of_range = start_of_range + assignment._chunk_size;
890 }
891
892 if (start_of_range >= end_of_range) {
893 continue;
894 }
895
896 // Old region in a young cycle or mixed cycle.
897 if (is_mixed) {
898 if (r->is_humongous()) {
899 // Need to examine both dirty and clean cards during mixed evac.
900 r->oop_iterate_humongous_slice_all(&cl,start_of_range, assignment._chunk_size);
901 } else {
902 // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced
903 // and filled. This will use mark bits to find objects that need to be updated.
904 update_references_in_old_region(cl, ctx, scanner, r, start_of_range, end_of_range);
905 }
906 } else {
907 // This is a young evacuation
908 size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster;
909 size_t clusters = assignment._chunk_size / cluster_size;
910 assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries");
911 scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, worker_id);
912 }
913
914 if (ShenandoahPacing) {
915 _heap->pacer()->report_updaterefs(pointer_delta(end_of_range, start_of_range));
916 }
917 }
918 }
919 }
920
921 template<class T>
922 void update_references_in_old_region(T &cl, const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner,
923 const ShenandoahHeapRegion* r, HeapWord* start_of_range,
924 HeapWord* end_of_range) const {
925 // In case last object in my range spans boundary of my chunk, I may need to scan all the way to top()
926 ShenandoahObjectToOopBoundedClosure<T> objs(&cl, start_of_range, r->top());
927
928 // Any object that begins in a previous range is part of a different scanning assignment. Any object that
929 // starts after end_of_range is also not my responsibility. (Either allocated during evacuation, so does
930 // not hold pointers to from-space, or is beyond the range of my assigned work chunk.)
931
932 // Find the first object that begins in my range, if there is one. Note that `p` will be set to `end_of_range`
933 // when no live object is found in the range.
934 HeapWord* tams = ctx->top_at_mark_start(r);
935 HeapWord* p = get_first_object_start_word(ctx, scanner, tams, start_of_range, end_of_range);
936
937 while (p < end_of_range) {
938 // p is known to point to the beginning of marked object obj
939 oop obj = cast_to_oop(p);
940 objs.do_object(obj);
941 HeapWord* prev_p = p;
942 p += obj->size();
943 if (p < tams) {
944 p = ctx->get_next_marked_addr(p, tams);
945 // If there are no more marked objects before tams, this returns tams. Note that tams is
946 // either >= end_of_range, or tams is the start of an object that is marked.
947 }
948 assert(p != prev_p, "Lack of forward progress");
949 }
950 }
951
952 HeapWord* get_first_object_start_word(const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner, HeapWord* tams,
953 HeapWord* start_of_range, HeapWord* end_of_range) const {
954 HeapWord* p = start_of_range;
955
956 if (p >= tams) {
957 // We cannot use ctx->is_marked(obj) to test whether an object begins at this address. Instead,
958 // we need to use the remembered set crossing map to advance p to the first object that starts
959 // within the enclosing card.
960 size_t card_index = scanner->card_index_for_addr(start_of_range);
961 while (true) {
962 HeapWord* first_object = scanner->first_object_in_card(card_index);
963 if (first_object != nullptr) {
964 p = first_object;
965 break;
966 } else if (scanner->addr_for_card_index(card_index + 1) < end_of_range) {
967 card_index++;
968 } else {
969 // Signal that no object was found in range
970 p = end_of_range;
971 break;
972 }
973 }
974 } else if (!ctx->is_marked(cast_to_oop(p))) {
975 p = ctx->get_next_marked_addr(p, tams);
976 // If there are no more marked objects before tams, this returns tams.
977 // Note that tams is either >= end_of_range, or tams is the start of an object that is marked.
978 }
979 return p;
980 }
981 };
982
983 void ShenandoahGenerationalHeap::update_heap_references(bool concurrent) {
984 assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
985 const uint nworkers = workers()->active_workers();
986 ShenandoahRegionChunkIterator work_list(nworkers);
987 if (concurrent) {
988 ShenandoahGenerationalUpdateHeapRefsTask<true> task(&_update_refs_iterator, &work_list);
989 workers()->run_task(&task);
990 } else {
991 ShenandoahGenerationalUpdateHeapRefsTask<false> task(&_update_refs_iterator, &work_list);
992 workers()->run_task(&task);
993 }
994
995 if (ShenandoahEnableCardStats) {
996 // Only do this if we are collecting card stats
997 ShenandoahScanRemembered* card_scan = old_generation()->card_scan();
998 assert(card_scan != nullptr, "Card table must exist when card stats are enabled");
999 card_scan->log_card_stats(nworkers, CARD_STAT_UPDATE_REFS);
1000 }
1001 }
1002
1003 namespace ShenandoahCompositeRegionClosure {
1004 template<typename C1, typename C2>
1005 class Closure : public ShenandoahHeapRegionClosure {
1006 private:
1007 C1 &_c1;
1008 C2 &_c2;
1009
1010 public:
1011 Closure(C1 &c1, C2 &c2) : ShenandoahHeapRegionClosure(), _c1(c1), _c2(c2) {}
1012
1013 void heap_region_do(ShenandoahHeapRegion* r) override {
1014 _c1.heap_region_do(r);
1015 _c2.heap_region_do(r);
1016 }
1017
1018 bool is_thread_safe() override {
1019 return _c1.is_thread_safe() && _c2.is_thread_safe();
1020 }
1021 };
1022
1023
1024 template<typename C1, typename C2>
1025 Closure<C1, C2> of(C1 &c1, C2 &c2) {
1026 return Closure<C1, C2>(c1, c2);
1027 }
1028 }
1029
1030 class ShenandoahUpdateRegionAges : public ShenandoahHeapRegionClosure {
1031 private:
1032 ShenandoahMarkingContext* _ctx;
1033
1034 public:
1035 explicit ShenandoahUpdateRegionAges(ShenandoahMarkingContext* ctx) : _ctx(ctx) { }
1036
1037 void heap_region_do(ShenandoahHeapRegion* r) override {
1038 // Maintenance of region age must follow evacuation in order to account for
1039 // evacuation allocations within survivor regions. We consult region age during
1040 // the subsequent evacuation to determine whether certain objects need to
1041 // be promoted.
1042 if (r->is_young() && r->is_active()) {
1043 HeapWord *tams = _ctx->top_at_mark_start(r);
1044 HeapWord *top = r->top();
1045
1046 // Allocations move the watermark when top moves. However, compacting
1047 // objects will sometimes lower top beneath the watermark, after which,
1048 // attempts to read the watermark will assert out (watermark should not be
1049 // higher than top).
1050 if (top > tams) {
1051 // There have been allocations in this region since the start of the cycle.
1052 // Any objects new to this region must not assimilate elevated age.
1053 r->reset_age();
1054 } else if (ShenandoahGenerationalHeap::heap()->is_aging_cycle()) {
1055 r->increment_age();
1056 }
1057 }
1058 }
1059
1060 bool is_thread_safe() override {
1061 return true;
1062 }
1063 };
1064
1065 void ShenandoahGenerationalHeap::final_update_refs_update_region_states() {
1066 ShenandoahSynchronizePinnedRegionStates pins;
1067 ShenandoahUpdateRegionAges ages(active_generation()->complete_marking_context());
1068 auto cl = ShenandoahCompositeRegionClosure::of(pins, ages);
1069 parallel_heap_region_iterate(&cl);
1070 }
1071
1072 void ShenandoahGenerationalHeap::complete_degenerated_cycle() {
1073 shenandoah_assert_heaplocked_or_safepoint();
1074 if (is_concurrent_old_mark_in_progress()) {
1075 // This is still necessary for degenerated cycles because the degeneration point may occur
1076 // after final mark of the young generation. See ShenandoahConcurrentGC::op_final_updaterefs for
1077 // a more detailed explanation.
1078 old_generation()->transfer_pointers_from_satb();
1079 }
1080
1081 // We defer generation resizing actions until after cset regions have been recycled.
1082 TransferResult result = balance_generations();
1083 LogTarget(Info, gc, ergo) lt;
1084 if (lt.is_enabled()) {
1085 LogStream ls(lt);
1086 result.print_on("Degenerated GC", &ls);
1087 }
1088
1089 // In case degeneration interrupted concurrent evacuation or update references, we need to clean up
1090 // transient state. Otherwise, these actions have no effect.
1091 reset_generation_reserves();
1092
1093 if (!old_generation()->is_parsable()) {
1094 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_coalesce_and_fill);
1095 coalesce_and_fill_old_regions(false);
1096 }
1097 }
1098
1099 void ShenandoahGenerationalHeap::complete_concurrent_cycle() {
1100 if (!old_generation()->is_parsable()) {
1101 // Class unloading may render the card offsets unusable, so we must rebuild them before
1102 // the next remembered set scan. We _could_ let the control thread do this sometime after
1103 // the global cycle has completed and before the next young collection, but under memory
1104 // pressure the control thread may not have the time (that is, because it's running back
1105 // to back GCs). In that scenario, we would have to make the old regions parsable before
1106 // we could start a young collection. This could delay the start of the young cycle and
1107 // throw off the heuristics.
1108 entry_global_coalesce_and_fill();
1109 }
1110
1111 TransferResult result;
1112 {
1113 ShenandoahHeapLocker locker(lock());
1114
1115 result = balance_generations();
1116 reset_generation_reserves();
1117 }
1118
1119 LogTarget(Info, gc, ergo) lt;
1120 if (lt.is_enabled()) {
1121 LogStream ls(lt);
1122 result.print_on("Concurrent GC", &ls);
1123 }
1124 }
1125
1126 void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() {
1127 const char* msg = "Coalescing and filling old regions";
1128 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
1129
1130 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
1131 EventMark em("%s", msg);
1132 ShenandoahWorkerScope scope(workers(),
1133 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
1134 "concurrent coalesce and fill");
1135
1136 coalesce_and_fill_old_regions(true);
1137 }
1138
1139 void ShenandoahGenerationalHeap::update_region_ages(ShenandoahMarkingContext* ctx) {
1140 ShenandoahUpdateRegionAges cl(ctx);
1141 parallel_heap_region_iterate(&cl);
1142 }