1 /*
  2  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 
 26 #include "precompiled.hpp"
 27 
 28 #include "gc/shared/strongRootsScope.hpp"
 29 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
 30 #include "gc/shenandoah/shenandoahAsserts.hpp"
 31 #include "gc/shenandoah/shenandoahCardTable.hpp"
 32 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 33 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 34 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 35 #include "gc/shenandoah/shenandoahHeap.hpp"
 36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 38 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
 39 #include "gc/shenandoah/shenandoahMarkClosures.hpp"
 40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
 41 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
 42 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
 44 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 45 #include "gc/shenandoah/shenandoahUtils.hpp"
 46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
 47 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
 48 #include "runtime/threads.hpp"
 49 #include "utilities/events.hpp"
 50 
 51 class ShenandoahFlushAllSATB : public ThreadClosure {
 52 private:
 53   SATBMarkQueueSet& _satb_qset;
 54 
 55 public:
 56   explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) :
 57     _satb_qset(satb_qset) {}
 58 
 59   void do_thread(Thread* thread) override {
 60     // Transfer any partial buffer to the qset for completed buffer processing.
 61     _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
 62   }
 63 };
 64 
 65 class ShenandoahProcessOldSATB : public SATBBufferClosure {
 66 private:
 67   ShenandoahObjToScanQueue*       _queue;
 68   ShenandoahHeap*                 _heap;
 69   ShenandoahMarkingContext* const _mark_context;
 70   size_t                          _trashed_oops;
 71 
 72 public:
 73   explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) :
 74     _queue(q),
 75     _heap(ShenandoahHeap::heap()),
 76     _mark_context(_heap->marking_context()),
 77     _trashed_oops(0) {}
 78 
 79   void do_buffer(void** buffer, size_t size) override {
 80     assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here");
 81     for (size_t i = 0; i < size; ++i) {
 82       oop *p = (oop *) &buffer[i];
 83       ShenandoahHeapRegion* region = _heap->heap_region_containing(*p);
 84       if (region->is_old() && region->is_active()) {
 85           ShenandoahMark::mark_through_ref<oop, OLD>(p, _queue, nullptr, _mark_context, false);
 86       } else {
 87         _trashed_oops++;
 88       }
 89     }
 90   }
 91 
 92   size_t trashed_oops() const {
 93     return _trashed_oops;
 94   }
 95 };
 96 
 97 class ShenandoahPurgeSATBTask : public WorkerTask {
 98 private:
 99   ShenandoahObjToScanQueueSet* _mark_queues;
100   volatile size_t             _trashed_oops;
101 
102 public:
103   explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) :
104     WorkerTask("Purge SATB"),
105     _mark_queues(queues),
106     _trashed_oops(0) {
107     Threads::change_thread_claim_token();
108   }
109 
110   ~ShenandoahPurgeSATBTask() {
111     if (_trashed_oops > 0) {
112       log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers", _trashed_oops);
113     }
114   }
115 
116   void work(uint worker_id) override {
117     ShenandoahParallelWorkerSession worker_session(worker_id);
118     ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
119     ShenandoahFlushAllSATB flusher(satb_queues);
120     Threads::possibly_parallel_threads_do(true /* is_par */, &flusher);
121 
122     ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id);
123     ShenandoahProcessOldSATB processor(mark_queue);
124     while (satb_queues.apply_closure_to_completed_buffer(&processor)) {}
125 
126     Atomic::add(&_trashed_oops, processor.trashed_oops());
127   }
128 };
129 
130 class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask {
131 private:
132   uint                    _nworkers;
133   ShenandoahHeapRegion**  _coalesce_and_fill_region_array;
134   uint                    _coalesce_and_fill_region_count;
135   volatile bool           _is_preempted;
136 
137 public:
138   ShenandoahConcurrentCoalesceAndFillTask(uint nworkers,
139                                           ShenandoahHeapRegion** coalesce_and_fill_region_array,
140                                           uint region_count) :
141     WorkerTask("Shenandoah Concurrent Coalesce and Fill"),
142     _nworkers(nworkers),
143     _coalesce_and_fill_region_array(coalesce_and_fill_region_array),
144     _coalesce_and_fill_region_count(region_count),
145     _is_preempted(false) {
146   }
147 
148   void work(uint worker_id) override {
149     ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_coalesce_and_fill, ShenandoahPhaseTimings::ScanClusters, worker_id);
150     for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) {
151       ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx];
152       if (r->is_humongous()) {
153         // There is only one object in this region and it is not garbage,
154         // so no need to coalesce or fill.
155         continue;
156       }
157 
158       if (!r->oop_coalesce_and_fill(true)) {
159         // Coalesce and fill has been preempted
160         Atomic::store(&_is_preempted, true);
161         return;
162       }
163     }
164   }
165 
166   // Value returned from is_completed() is only valid after all worker thread have terminated.
167   bool is_completed() {
168     return !Atomic::load(&_is_preempted);
169   }
170 };
171 
172 ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity)
173   : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity),
174     _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)),
175     _old_heuristics(nullptr),
176     _region_balance(0),
177     _promoted_reserve(0),
178     _promoted_expended(0),
179     _promotion_potential(0),
180     _pad_for_promote_in_place(0),
181     _promotable_humongous_regions(0),
182     _promotable_regular_regions(0),
183     _is_parsable(true),
184     _card_scan(nullptr),
185     _state(WAITING_FOR_BOOTSTRAP),
186     _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION),
187     _min_growth_before_compaction ((ShenandoahMinOldGenGrowthPercent * FRACTIONAL_DENOMINATOR) / 100)
188 {
189   _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
190   // Always clear references for old generation
191   ref_processor()->set_soft_reference_policy(true);
192 
193   if (ShenandoahCardBarrier) {
194     ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table();
195     size_t card_count = card_table->cards_required(ShenandoahHeap::heap()->reserved_region().word_size());
196     auto rs = new ShenandoahDirectCardMarkRememberedSet(card_table, card_count);
197     _card_scan = new ShenandoahScanRemembered(rs);
198   }
199 }
200 
201 void ShenandoahOldGeneration::set_promoted_reserve(size_t new_val) {
202   shenandoah_assert_heaplocked_or_safepoint();
203   _promoted_reserve = new_val;
204 }
205 
206 size_t ShenandoahOldGeneration::get_promoted_reserve() const {
207   return _promoted_reserve;
208 }
209 
210 void ShenandoahOldGeneration::augment_promoted_reserve(size_t increment) {
211   shenandoah_assert_heaplocked_or_safepoint();
212   _promoted_reserve += increment;
213 }
214 
215 void ShenandoahOldGeneration::reset_promoted_expended() {
216   shenandoah_assert_heaplocked_or_safepoint();
217   Atomic::store(&_promoted_expended, (size_t) 0);
218 }
219 
220 size_t ShenandoahOldGeneration::expend_promoted(size_t increment) {
221   shenandoah_assert_heaplocked_or_safepoint();
222   assert(get_promoted_expended() + increment <= get_promoted_reserve(), "Do not expend more promotion than budgeted");
223   return Atomic::add(&_promoted_expended, increment);
224 }
225 
226 size_t ShenandoahOldGeneration::unexpend_promoted(size_t decrement) {
227   return Atomic::sub(&_promoted_expended, decrement);
228 }
229 
230 size_t ShenandoahOldGeneration::get_promoted_expended() const {
231   return Atomic::load(&_promoted_expended);
232 }
233 
234 bool ShenandoahOldGeneration::can_allocate(const ShenandoahAllocRequest &req) const {
235   assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory");
236 
237   const size_t requested_bytes = req.size() * HeapWordSize;
238   // The promotion reserve may also be used for evacuations. If we can promote this object,
239   // then we can also evacuate it.
240   if (can_promote(requested_bytes)) {
241     // The promotion reserve should be able to accommodate this request. The request
242     // might still fail if alignment with the card table increases the size. The request
243     // may also fail if the heap is badly fragmented and the free set cannot find room for it.
244     return true;
245   }
246 
247   if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
248     // The promotion reserve cannot accommodate this plab request. Check if we still have room for
249     // evacuations. Note that we cannot really know how much of the plab will be used for evacuations,
250     // so here we only check that some evacuation reserve still exists.
251     return get_evacuation_reserve() > 0;
252   }
253 
254   // This is a shared allocation request. We've already checked that it can't be promoted, so if
255   // it is a promotion, we return false. Otherwise, it is a shared evacuation request, and we allow
256   // the allocation to proceed.
257   return !req.is_promotion();
258 }
259 
260 void
261 ShenandoahOldGeneration::configure_plab_for_current_thread(const ShenandoahAllocRequest &req) {
262   // Note: Even when a mutator is performing a promotion outside a LAB, we use a 'shared_gc' request.
263   if (req.is_gc_alloc()) {
264     const size_t actual_size = req.actual_size() * HeapWordSize;
265     if (req.type() ==  ShenandoahAllocRequest::_alloc_plab) {
266       // We've created a new plab. Now we configure it whether it will be used for promotions
267       // and evacuations - or just evacuations.
268       Thread* thread = Thread::current();
269       ShenandoahThreadLocalData::reset_plab_promoted(thread);
270 
271       // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries).
272       // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread.
273       if (can_promote(actual_size)) {
274         // Assume the entirety of this PLAB will be used for promotion.  This prevents promotion from overreach.
275         // When we retire this plab, we'll unexpend what we don't really use.
276         expend_promoted(actual_size);
277         ShenandoahThreadLocalData::enable_plab_promotions(thread);
278         ShenandoahThreadLocalData::set_plab_actual_size(thread, actual_size);
279       } else {
280         // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations.
281         ShenandoahThreadLocalData::disable_plab_promotions(thread);
282         ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
283       }
284     } else if (req.is_promotion()) {
285       // Shared promotion.
286       expend_promoted(actual_size);
287     }
288   }
289 }
290 
291 size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const {
292   return _live_bytes_after_last_mark;
293 }
294 
295 void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) {
296   if (bytes == 0) {
297     // Restart search for best old-gen size to the initial state
298     _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
299     _growth_before_compaction = INITIAL_GROWTH_BEFORE_COMPACTION;
300   } else {
301     _live_bytes_after_last_mark = bytes;
302     _growth_before_compaction /= 2;
303     if (_growth_before_compaction < _min_growth_before_compaction) {
304       _growth_before_compaction = _min_growth_before_compaction;
305     }
306   }
307 }
308 
309 void ShenandoahOldGeneration::handle_failed_transfer() {
310   _old_heuristics->trigger_cannot_expand();
311 }
312 
313 size_t ShenandoahOldGeneration::usage_trigger_threshold() const {
314   size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR;
315   return result;
316 }
317 
318 bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const {
319   return region->is_old();
320 }
321 
322 void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
323   ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
324   ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions_cl);
325 }
326 
327 void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
328   ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
329   ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl);
330 }
331 
332 void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) {
333   ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress);
334 }
335 
336 bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() {
337   return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress();
338 }
339 
340 void ShenandoahOldGeneration::cancel_marking() {
341   if (is_concurrent_mark_in_progress()) {
342     log_info(gc)("Abandon SATB buffers");
343     ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
344   }
345 
346   ShenandoahGeneration::cancel_marking();
347 }
348 
349 void ShenandoahOldGeneration::cancel_gc() {
350   shenandoah_assert_safepoint();
351   if (is_idle()) {
352 #ifdef ASSERT
353     validate_waiting_for_bootstrap();
354 #endif
355   } else {
356     log_info(gc)("Terminating old gc cycle.");
357     // Stop marking
358     cancel_marking();
359     // Stop tracking old regions
360     abandon_collection_candidates();
361     // Remove old generation access to young generation mark queues
362     ShenandoahHeap::heap()->young_generation()->set_old_gen_task_queues(nullptr);
363     // Transition to IDLE now.
364     transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
365   }
366 }
367 
368 void ShenandoahOldGeneration::prepare_gc() {
369   // Now that we have made the old generation parsable, it is safe to reset the mark bitmap.
370   assert(state() != FILLING, "Cannot reset old without making it parsable");
371 
372   ShenandoahGeneration::prepare_gc();
373 }
374 
375 bool ShenandoahOldGeneration::entry_coalesce_and_fill() {
376   ShenandoahHeap* const heap = ShenandoahHeap::heap();
377 
378   static const char* msg = "Coalescing and filling (OLD)";
379   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
380 
381   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
382   EventMark em("%s", msg);
383   ShenandoahWorkerScope scope(heap->workers(),
384                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
385                               msg);
386 
387   return coalesce_and_fill();
388 }
389 
390 // Make the old generation regions parsable, so they can be safely
391 // scanned when looking for objects in memory indicated by dirty cards.
392 bool ShenandoahOldGeneration::coalesce_and_fill() {
393   transition_to(FILLING);
394 
395   // This code will see the same set of regions to fill on each resumption as it did
396   // on the initial run. That's okay because each region keeps track of its own coalesce
397   // and fill state. Regions that were filled on a prior attempt will not try to fill again.
398   uint coalesce_and_fill_regions_count = _old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array);
399   assert(coalesce_and_fill_regions_count <= ShenandoahHeap::heap()->num_regions(), "Sanity");
400   if (coalesce_and_fill_regions_count == 0) {
401     // No regions need to be filled.
402     abandon_collection_candidates();
403     return true;
404   }
405 
406   ShenandoahHeap* const heap = ShenandoahHeap::heap();
407   WorkerThreads* workers = heap->workers();
408   uint nworkers = workers->active_workers();
409   ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count);
410 
411   log_info(gc)("Starting (or resuming) coalesce-and-fill of " UINT32_FORMAT " old heap regions", coalesce_and_fill_regions_count);
412   workers->run_task(&task);
413   if (task.is_completed()) {
414     // We no longer need to track regions that need to be coalesced and filled.
415     abandon_collection_candidates();
416     return true;
417   } else {
418     // Coalesce-and-fill has been preempted. We'll finish that effort in the future.  Do not invoke
419     // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap
420     // and invokes set_mark_incomplete().  Coalesce-and-fill depends on the mark bitmap.
421     log_debug(gc)("Suspending coalesce-and-fill of old heap regions");
422     return false;
423   }
424 }
425 
426 void ShenandoahOldGeneration::transfer_pointers_from_satb() {
427   ShenandoahHeap* heap = ShenandoahHeap::heap();
428   shenandoah_assert_safepoint();
429   assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking.");
430   log_info(gc)("Transfer SATB buffers");
431   uint nworkers = heap->workers()->active_workers();
432   StrongRootsScope scope(nworkers);
433 
434   ShenandoahPurgeSATBTask purge_satb_task(task_queues());
435   heap->workers()->run_task(&purge_satb_task);
436 }
437 
438 bool ShenandoahOldGeneration::contains(oop obj) const {
439   return ShenandoahHeap::heap()->is_in_old(obj);
440 }
441 
442 void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) {
443   ShenandoahHeap* heap = ShenandoahHeap::heap();
444   assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
445 
446   {
447     ShenandoahGCPhase phase(concurrent ?
448         ShenandoahPhaseTimings::final_update_region_states :
449         ShenandoahPhaseTimings::degen_gc_final_update_region_states);
450     ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
451 
452     parallel_heap_region_iterate(&cl);
453     heap->assert_pinned_region_status();
454   }
455 
456   {
457     // This doesn't actually choose a collection set, but prepares a list of
458     // regions as 'candidates' for inclusion in a mixed collection.
459     ShenandoahGCPhase phase(concurrent ?
460         ShenandoahPhaseTimings::choose_cset :
461         ShenandoahPhaseTimings::degen_gc_choose_cset);
462     ShenandoahHeapLocker locker(heap->lock());
463     _old_heuristics->prepare_for_old_collections();
464   }
465 
466   {
467     // Though we did not choose a collection set above, we still may have
468     // freed up immediate garbage regions so proceed with rebuilding the free set.
469     ShenandoahGCPhase phase(concurrent ?
470         ShenandoahPhaseTimings::final_rebuild_freeset :
471         ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
472     ShenandoahHeapLocker locker(heap->lock());
473     size_t cset_young_regions, cset_old_regions;
474     size_t first_old, last_old, num_old;
475     heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old);
476     // This is just old-gen completion.  No future budgeting required here.  The only reason to rebuild the freeset here
477     // is in case there was any immediate old garbage identified.
478     heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old);
479   }
480 }
481 
482 const char* ShenandoahOldGeneration::state_name(State state) {
483   switch (state) {
484     case WAITING_FOR_BOOTSTRAP:   return "Waiting for Bootstrap";
485     case FILLING:                 return "Coalescing";
486     case BOOTSTRAPPING:           return "Bootstrapping";
487     case MARKING:                 return "Marking";
488     case EVACUATING:              return "Evacuating";
489     case EVACUATING_AFTER_GLOBAL: return "Evacuating (G)";
490     default:
491       ShouldNotReachHere();
492       return "Unknown";
493   }
494 }
495 
496 void ShenandoahOldGeneration::transition_to(State new_state) {
497   if (_state != new_state) {
498     log_info(gc)("Old generation transition from %s to %s", state_name(_state), state_name(new_state));
499     validate_transition(new_state);
500     _state = new_state;
501   }
502 }
503 
504 #ifdef ASSERT
505 // This diagram depicts the expected state transitions for marking the old generation
506 // and preparing for old collections. When a young generation cycle executes, the
507 // remembered set scan must visit objects in old regions. Visiting an object which
508 // has become dead on previous old cycles will result in crashes. To avoid visiting
509 // such objects, the remembered set scan will use the old generation mark bitmap when
510 // possible. It is _not_ possible to use the old generation bitmap when old marking
511 // is active (bitmap is not complete). For this reason, the old regions are made
512 // parsable _before_ the old generation bitmap is reset. The diagram does not depict
513 // cancellation of old collections by global or full collections.
514 //
515 // When a global collection supersedes an old collection, the global mark still
516 // "completes" the old mark bitmap. Subsequent remembered set scans may use the
517 // old generation mark bitmap, but any uncollected old regions must still be made parsable
518 // before the next old generation cycle begins. For this reason, a global collection may
519 // create mixed collection candidates and coalesce and fill candidates and will put
520 // the old generation in the respective states (EVACUATING or FILLING). After a Full GC,
521 // the mark bitmaps are all reset, all regions are parsable and the mark context will
522 // not be "complete". After a Full GC, remembered set scans will _not_ use the mark bitmap
523 // and we expect the old generation to be waiting for bootstrap.
524 //
525 //                              +-----------------+
526 //               +------------> |     FILLING     | <---+
527 //               |   +--------> |                 |     |
528 //               |   |          +-----------------+     |
529 //               |   |            |                     |
530 //               |   |            | Filling Complete    | <-> A global collection may
531 //               |   |            v                     |     move the old generation
532 //               |   |          +-----------------+     |     directly from waiting for
533 //           +-- |-- |--------> |     WAITING     |     |     bootstrap to filling or
534 //           |   |   |    +---- |  FOR BOOTSTRAP  | ----+     evacuating. It may also
535 //           |   |   |    |     +-----------------+           move from filling to waiting
536 //           |   |   |    |       |                           for bootstrap.
537 //           |   |   |    |       | Reset Bitmap
538 //           |   |   |    |       v
539 //           |   |   |    |     +-----------------+     +----------------------+
540 //           |   |   |    |     |    BOOTSTRAP    | <-> |       YOUNG GC       |
541 //           |   |   |    |     |                 |     | (RSet Parses Region) |
542 //           |   |   |    |     +-----------------+     +----------------------+
543 //           |   |   |    |       |
544 //           |   |   |    |       | Old Marking
545 //           |   |   |    |       v
546 //           |   |   |    |     +-----------------+     +----------------------+
547 //           |   |   |    |     |     MARKING     | <-> |       YOUNG GC       |
548 //           |   |   +--------- |                 |     | (RSet Parses Region) |
549 //           |   |        |     +-----------------+     +----------------------+
550 //           |   |        |       |
551 //           |   |        |       | Has Evacuation Candidates
552 //           |   |        |       v
553 //           |   |        |     +-----------------+     +--------------------+
554 //           |   |        +---> |    EVACUATING   | <-> |      YOUNG GC      |
555 //           |   +------------- |                 |     | (RSet Uses Bitmap) |
556 //           |                  +-----------------+     +--------------------+
557 //           |                    |
558 //           |                    | Global Cycle Coalesces and Fills Old Regions
559 //           |                    v
560 //           |                  +-----------------+     +--------------------+
561 //           +----------------- |    EVACUATING   | <-> |      YOUNG GC      |
562 //                              |   AFTER GLOBAL  |     | (RSet Uses Bitmap) |
563 //                              +-----------------+     +--------------------+
564 //
565 //
566 void ShenandoahOldGeneration::validate_transition(State new_state) {
567   ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
568   switch (new_state) {
569     case FILLING:
570       assert(_state != BOOTSTRAPPING, "Cannot begin making old regions parsable after bootstrapping");
571       assert(is_mark_complete(), "Cannot begin filling without first completing marking, state is '%s'", state_name(_state));
572       assert(_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot begin filling without something to fill.");
573       break;
574     case WAITING_FOR_BOOTSTRAP:
575       // GC cancellation can send us back here from any state.
576       validate_waiting_for_bootstrap();
577       break;
578     case BOOTSTRAPPING:
579       assert(_state == WAITING_FOR_BOOTSTRAP, "Cannot reset bitmap without making old regions parsable, state is '%s'", state_name(_state));
580       assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates");
581       assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parsable.");
582       break;
583     case MARKING:
584       assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state));
585       assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues.");
586       assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now.");
587       break;
588     case EVACUATING_AFTER_GLOBAL:
589       assert(_state == EVACUATING, "Must have been evacuating, state is '%s'", state_name(_state));
590       break;
591     case EVACUATING:
592       assert(_state == WAITING_FOR_BOOTSTRAP || _state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state));
593       assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here.");
594       break;
595     default:
596       fatal("Unknown new state");
597   }
598 }
599 
600 bool ShenandoahOldGeneration::validate_waiting_for_bootstrap() {
601   ShenandoahHeap* heap = ShenandoahHeap::heap();
602   assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become ready for bootstrap during old mark.");
603   assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become ready for bootstrap when still setup for bootstrapping.");
604   assert(!is_concurrent_mark_in_progress(), "Cannot be marking in IDLE");
605   assert(!heap->young_generation()->is_bootstrap_cycle(), "Cannot have old mark queues if IDLE");
606   assert(!_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot have coalesce and fill candidates in IDLE");
607   assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot have mixed collection candidates in IDLE");
608   return true;
609 }
610 #endif
611 
612 ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
613   _old_heuristics = new ShenandoahOldHeuristics(this, ShenandoahGenerationalHeap::heap());
614   _old_heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval);
615   _heuristics = _old_heuristics;
616   return _heuristics;
617 }
618 
619 void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) {
620   heuristics()->record_success_concurrent();
621   ShenandoahHeap::heap()->shenandoah_policy()->record_success_old();
622 }
623 
624 void ShenandoahOldGeneration::handle_failed_evacuation() {
625   if (_failed_evacuation.try_set()) {
626     log_info(gc)("Old gen evac failure.");
627   }
628 }
629 
630 void ShenandoahOldGeneration::handle_failed_promotion(Thread* thread, size_t size) {
631   // We squelch excessive reports to reduce noise in logs.
632   const size_t MaxReportsPerEpoch = 4;
633   static size_t last_report_epoch = 0;
634   static size_t epoch_report_count = 0;
635   auto heap = ShenandoahGenerationalHeap::heap();
636 
637   size_t promotion_reserve;
638   size_t promotion_expended;
639 
640   const size_t gc_id = heap->control_thread()->get_gc_id();
641 
642   if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) {
643     {
644       // Promotion failures should be very rare.  Invest in providing useful diagnostic info.
645       ShenandoahHeapLocker locker(heap->lock());
646       promotion_reserve = get_promoted_reserve();
647       promotion_expended = get_promoted_expended();
648     }
649     PLAB* const plab = ShenandoahThreadLocalData::plab(thread);
650     const size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining();
651     const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled";
652 
653     log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT
654                        ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT
655                        ", old capacity: " SIZE_FORMAT ", old_used: " SIZE_FORMAT ", old unaffiliated regions: " SIZE_FORMAT,
656                        size * HeapWordSize, plab == nullptr? "no": "yes",
657                        words_remaining * HeapWordSize, promote_enabled, promotion_reserve, promotion_expended,
658                        max_capacity(), used(), free_unaffiliated_regions());
659 
660     if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) {
661       log_info(gc, ergo)("Squelching additional promotion failure reports for current epoch");
662     } else if (gc_id != last_report_epoch) {
663       last_report_epoch = gc_id;
664       epoch_report_count = 1;
665     }
666   }
667 }
668 
669 void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, bool promotion) {
670   // Only register the copy of the object that won the evacuation race.
671   _card_scan->register_object_without_lock(obj);
672 
673   // Mark the entire range of the evacuated object as dirty.  At next remembered set scan,
674   // we will clear dirty bits that do not hold interesting pointers.  It's more efficient to
675   // do this in batch, in a background GC thread than to try to carefully dirty only cards
676   // that hold interesting pointers right now.
677   _card_scan->mark_range_as_dirty(obj, words);
678 
679   if (promotion) {
680     // This evacuation was a promotion, track this as allocation against old gen
681     increase_allocated(words * HeapWordSize);
682   }
683 }
684 
685 bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() {
686   return _old_heuristics->unprocessed_old_collection_candidates() > 0;
687 }
688 
689 size_t ShenandoahOldGeneration::unprocessed_collection_candidates_live_memory() {
690   return _old_heuristics->unprocessed_old_collection_candidates_live_memory();
691 }
692 
693 void ShenandoahOldGeneration::abandon_collection_candidates() {
694   _old_heuristics->abandon_collection_candidates();
695 }
696 
697 void ShenandoahOldGeneration::prepare_for_mixed_collections_after_global_gc() {
698   assert(is_mark_complete(), "Expected old generation mark to be complete after global cycle.");
699   _old_heuristics->prepare_for_old_collections();
700   log_info(gc)("After choosing global collection set, mixed candidates: " UINT32_FORMAT ", coalescing candidates: " SIZE_FORMAT,
701                _old_heuristics->unprocessed_old_collection_candidates(),
702                _old_heuristics->coalesce_and_fill_candidates_count());
703 }
704 
705 void ShenandoahOldGeneration::parallel_region_iterate_free(ShenandoahHeapRegionClosure* cl) {
706   // Iterate over old and free regions (exclude young).
707   ShenandoahExcludeRegionClosure<YOUNG_GENERATION> exclude_cl(cl);
708   ShenandoahGeneration::parallel_region_iterate_free(&exclude_cl);
709 }
710 
711 void ShenandoahOldGeneration::set_parsable(bool parsable) {
712   _is_parsable = parsable;
713   if (_is_parsable) {
714     // The current state would have been chosen during final mark of the global
715     // collection, _before_ any decisions about class unloading have been made.
716     //
717     // After unloading classes, we have made the old generation regions parsable.
718     // We can skip filling or transition to a state that knows everything has
719     // already been filled.
720     switch (state()) {
721       case ShenandoahOldGeneration::EVACUATING:
722         transition_to(ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL);
723         break;
724       case ShenandoahOldGeneration::FILLING:
725         assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Expected no mixed collection candidates");
726         assert(_old_heuristics->coalesce_and_fill_candidates_count() > 0, "Expected coalesce and fill candidates");
727         // When the heuristic put the old generation in this state, it didn't know
728         // that we would unload classes and make everything parsable. But, we know
729         // that now so we can override this state.
730         abandon_collection_candidates();
731         transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
732         break;
733       default:
734         // We can get here during a full GC. The full GC will cancel anything
735         // happening in the old generation and return it to the waiting for bootstrap
736         // state. The full GC will then record that the old regions are parsable
737         // after rebuilding the remembered set.
738         assert(is_idle(), "Unexpected state %s at end of global GC", state_name());
739         break;
740     }
741   }
742 }
743 
744 void ShenandoahOldGeneration::complete_mixed_evacuations() {
745   assert(is_doing_mixed_evacuations(), "Mixed evacuations should be in progress");
746   if (!_old_heuristics->has_coalesce_and_fill_candidates()) {
747     // No candidate regions to coalesce and fill
748     transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
749     return;
750   }
751 
752   if (state() == ShenandoahOldGeneration::EVACUATING) {
753     transition_to(ShenandoahOldGeneration::FILLING);
754     return;
755   }
756 
757   // Here, we have no more candidates for mixed collections. The candidates for coalescing
758   // and filling have already been processed during the global cycle, so there is nothing
759   // more to do.
760   assert(state() == ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL, "Should be evacuating after a global cycle");
761   abandon_collection_candidates();
762   transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
763 }
764 
765 void ShenandoahOldGeneration::abandon_mixed_evacuations() {
766   switch(state()) {
767     case ShenandoahOldGeneration::EVACUATING:
768       transition_to(ShenandoahOldGeneration::FILLING);
769       break;
770     case ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL:
771       abandon_collection_candidates();
772       transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
773       break;
774     default:
775       ShouldNotReachHere();
776       break;
777   }
778 }
779 
780 void ShenandoahOldGeneration::clear_cards_for(ShenandoahHeapRegion* region) {
781   _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom()));
782 }
783 
784 void ShenandoahOldGeneration::mark_card_as_dirty(void* location) {
785   _card_scan->mark_card_as_dirty((HeapWord*)location);
786 }