1 /*
  2  * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
  4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  6  *
  7  * This code is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License version 2 only, as
  9  * published by the Free Software Foundation.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  *
 25  */
 26 
 27 #include "precompiled.hpp"
 28 #include "gc/shared/cardTable.hpp"
 29 #include "gc/shared/space.inline.hpp"
 30 #include "gc/shared/tlab_globals.hpp"
 31 #include "gc/shenandoah/shenandoahCardTable.hpp"
 32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 33 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
 34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 35 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 36 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 37 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
 38 #include "gc/shenandoah/shenandoahGeneration.hpp"
 39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
 40 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 41 #include "jfr/jfrEvents.hpp"
 42 #include "memory/allocation.hpp"
 43 #include "memory/iterator.inline.hpp"
 44 #include "memory/resourceArea.hpp"
 45 #include "memory/universe.hpp"
 46 #include "oops/oop.inline.hpp"
 47 #include "runtime/atomic.hpp"
 48 #include "runtime/globals_extension.hpp"
 49 #include "runtime/java.hpp"
 50 #include "runtime/mutexLocker.hpp"
 51 #include "runtime/os.hpp"
 52 #include "runtime/safepoint.hpp"
 53 #include "utilities/powerOfTwo.hpp"
 54 
 55 
 56 size_t ShenandoahHeapRegion::RegionCount = 0;
 57 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
 58 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
 59 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
 60 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
 61 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
 62 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
 63 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
 64 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
 65 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
 66 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
 67 
 68 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
 69   _index(index),
 70   _bottom(start),
 71   _end(start + RegionSizeWords),
 72   _new_top(nullptr),
 73   _empty_time(os::elapsedTime()),
 74   _top_before_promoted(nullptr),
 75   _state(committed ? _empty_committed : _empty_uncommitted),
 76   _top(start),
 77   _tlab_allocs(0),
 78   _gclab_allocs(0),
 79   _plab_allocs(0),
 80   _live_data(0),
 81   _critical_pins(0),
 82   _update_watermark(start),
 83   _age(0)
 84 #ifdef SHENANDOAH_CENSUS_NOISE
 85   , _youth(0)
 86 #endif // SHENANDOAH_CENSUS_NOISE
 87   {
 88 
 89   assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
 90          "invalid space boundaries");
 91   if (ZapUnusedHeapArea && committed) {
 92     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
 93   }
 94 }
 95 
 96 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
 97   stringStream ss;
 98   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
 99   print_on(&ss);
100   fatal("%s", ss.freeze());
101 }
102 
103 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) {
104   shenandoah_assert_heaplocked();
105   reset_age();
106   switch (_state) {
107     case _empty_uncommitted:
108       do_commit();
109     case _empty_committed:
110       assert(this->affiliation() == affiliation, "Region affiliation should already be established");
111       set_state(_regular);
112     case _regular:
113     case _pinned:
114       return;
115     default:
116       report_illegal_transition("regular allocation");
117   }
118 }
119 
120 // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned.  This implements
121 // behavior previously performed as a side effect of make_regular_bypass().
122 void ShenandoahHeapRegion::make_young_maybe() {
123   shenandoah_assert_heaplocked();
124   switch (_state) {
125    case _empty_uncommitted:
126    case _empty_committed:
127    case _cset:
128    case _humongous_start:
129    case _humongous_cont:
130      if (affiliation() != YOUNG_GENERATION) {
131        ShenandoahHeap* heap = ShenandoahHeap::heap();
132        if (heap->mode()->is_generational()) {
133          if (is_old()) {
134            heap->old_generation()->decrement_affiliated_region_count();
135          }
136          heap->young_generation()->increment_affiliated_region_count();
137        }
138        set_affiliation(YOUNG_GENERATION);
139      }
140      return;
141    case _pinned_cset:
142    case _regular:
143    case _pinned:
144      return;
145    default:
146      assert(false, "Unexpected _state in make_young_maybe");
147   }
148 }
149 
150 void ShenandoahHeapRegion::make_regular_bypass() {
151   shenandoah_assert_heaplocked();
152   assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
153           "only for full or degen GC");
154   reset_age();
155   switch (_state) {
156     case _empty_uncommitted:
157       do_commit();
158     case _empty_committed:
159     case _cset:
160     case _humongous_start:
161     case _humongous_cont:
162       set_state(_regular);
163       return;
164     case _pinned_cset:
165       set_state(_pinned);
166       return;
167     case _regular:
168     case _pinned:
169       return;
170     default:
171       report_illegal_transition("regular bypass");
172   }
173 }
174 
175 void ShenandoahHeapRegion::make_humongous_start() {
176   shenandoah_assert_heaplocked();
177   reset_age();
178   switch (_state) {
179     case _empty_uncommitted:
180       do_commit();
181     case _empty_committed:
182       set_state(_humongous_start);
183       return;
184     default:
185       report_illegal_transition("humongous start allocation");
186   }
187 }
188 
189 void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) {
190   shenandoah_assert_heaplocked();
191   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
192   // Don't bother to account for affiliated regions during Full GC.  We recompute totals at end.
193   set_affiliation(affiliation);
194   reset_age();
195   switch (_state) {
196     case _empty_committed:
197     case _regular:
198     case _humongous_start:
199     case _humongous_cont:
200       set_state(_humongous_start);
201       return;
202     default:
203       report_illegal_transition("humongous start bypass");
204   }
205 }
206 
207 void ShenandoahHeapRegion::make_humongous_cont() {
208   shenandoah_assert_heaplocked();
209   reset_age();
210   switch (_state) {
211     case _empty_uncommitted:
212       do_commit();
213     case _empty_committed:
214      set_state(_humongous_cont);
215       return;
216     default:
217       report_illegal_transition("humongous continuation allocation");
218   }
219 }
220 
221 void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) {
222   shenandoah_assert_heaplocked();
223   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
224   set_affiliation(affiliation);
225   // Don't bother to account for affiliated regions during Full GC.  We recompute totals at end.
226   reset_age();
227   switch (_state) {
228     case _empty_committed:
229     case _regular:
230     case _humongous_start:
231     case _humongous_cont:
232       set_state(_humongous_cont);
233       return;
234     default:
235       report_illegal_transition("humongous continuation bypass");
236   }
237 }
238 
239 void ShenandoahHeapRegion::make_pinned() {
240   shenandoah_assert_heaplocked();
241   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
242 
243   switch (_state) {
244     case _regular:
245       set_state(_pinned);
246     case _pinned_cset:
247     case _pinned:
248       return;
249     case _humongous_start:
250       set_state(_pinned_humongous_start);
251     case _pinned_humongous_start:
252       return;
253     case _cset:
254       _state = _pinned_cset;
255       return;
256     default:
257       report_illegal_transition("pinning");
258   }
259 }
260 
261 void ShenandoahHeapRegion::make_unpinned() {
262   shenandoah_assert_heaplocked();
263   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
264 
265   switch (_state) {
266     case _pinned:
267       assert(is_affiliated(), "Pinned region should be affiliated");
268       set_state(_regular);
269       return;
270     case _regular:
271     case _humongous_start:
272       return;
273     case _pinned_cset:
274       set_state(_cset);
275       return;
276     case _pinned_humongous_start:
277       set_state(_humongous_start);
278       return;
279     default:
280       report_illegal_transition("unpinning");
281   }
282 }
283 
284 void ShenandoahHeapRegion::make_cset() {
285   shenandoah_assert_heaplocked();
286   // Leave age untouched.  We need to consult the age when we are deciding whether to promote evacuated objects.
287   switch (_state) {
288     case _regular:
289       set_state(_cset);
290     case _cset:
291       return;
292     default:
293       report_illegal_transition("cset");
294   }
295 }
296 
297 void ShenandoahHeapRegion::make_trash() {
298   shenandoah_assert_heaplocked();
299   reset_age();
300   switch (_state) {
301     case _humongous_start:
302     case _humongous_cont:
303     {
304       // Reclaiming humongous regions and reclaim humongous waste.  When this region is eventually recycled, we'll reclaim
305       // its used memory.  At recycle time, we no longer recognize this as a humongous region.
306       decrement_humongous_waste();
307     }
308     case _cset:
309       // Reclaiming cset regions
310     case _regular:
311       // Immediate region reclaim
312       set_state(_trash);
313       return;
314     default:
315       report_illegal_transition("trashing");
316   }
317 }
318 
319 void ShenandoahHeapRegion::make_trash_immediate() {
320   make_trash();
321 
322   // On this path, we know there are no marked objects in the region,
323   // tell marking context about it to bypass bitmap resets.
324   assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here.");
325   ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this);
326 }
327 
328 void ShenandoahHeapRegion::make_empty() {
329   shenandoah_assert_heaplocked();
330   reset_age();
331   CENSUS_NOISE(clear_youth();)
332   switch (_state) {
333     case _trash:
334       set_state(_empty_committed);
335       _empty_time = os::elapsedTime();
336       return;
337     default:
338       report_illegal_transition("emptying");
339   }
340 }
341 
342 void ShenandoahHeapRegion::make_uncommitted() {
343   shenandoah_assert_heaplocked();
344   switch (_state) {
345     case _empty_committed:
346       do_uncommit();
347       set_state(_empty_uncommitted);
348       return;
349     default:
350       report_illegal_transition("uncommiting");
351   }
352 }
353 
354 void ShenandoahHeapRegion::make_committed_bypass() {
355   shenandoah_assert_heaplocked();
356   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
357 
358   switch (_state) {
359     case _empty_uncommitted:
360       do_commit();
361       set_state(_empty_committed);
362       return;
363     default:
364       report_illegal_transition("commit bypass");
365   }
366 }
367 
368 void ShenandoahHeapRegion::reset_alloc_metadata() {
369   _tlab_allocs = 0;
370   _gclab_allocs = 0;
371   _plab_allocs = 0;
372 }
373 
374 size_t ShenandoahHeapRegion::get_shared_allocs() const {
375   return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize;
376 }
377 
378 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
379   return _tlab_allocs * HeapWordSize;
380 }
381 
382 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
383   return _gclab_allocs * HeapWordSize;
384 }
385 
386 size_t ShenandoahHeapRegion::get_plab_allocs() const {
387   return _plab_allocs * HeapWordSize;
388 }
389 
390 void ShenandoahHeapRegion::set_live_data(size_t s) {
391   assert(Thread::current()->is_VM_thread(), "by VM thread");
392   _live_data = (s >> LogHeapWordSize);
393 }
394 
395 void ShenandoahHeapRegion::print_on(outputStream* st) const {
396   st->print("|");
397   st->print(SIZE_FORMAT_W(5), this->_index);
398 
399   switch (_state) {
400     case _empty_uncommitted:
401       st->print("|EU ");
402       break;
403     case _empty_committed:
404       st->print("|EC ");
405       break;
406     case _regular:
407       st->print("|R  ");
408       break;
409     case _humongous_start:
410       st->print("|H  ");
411       break;
412     case _pinned_humongous_start:
413       st->print("|HP ");
414       break;
415     case _humongous_cont:
416       st->print("|HC ");
417       break;
418     case _cset:
419       st->print("|CS ");
420       break;
421     case _trash:
422       st->print("|TR ");
423       break;
424     case _pinned:
425       st->print("|P  ");
426       break;
427     case _pinned_cset:
428       st->print("|CSP");
429       break;
430     default:
431       ShouldNotReachHere();
432   }
433 
434   st->print("|%s", shenandoah_affiliation_code(affiliation()));
435 
436 #define SHR_PTR_FORMAT "%12" PRIxPTR
437 
438   st->print("|BTE " SHR_PTR_FORMAT  ", " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT,
439             p2i(bottom()), p2i(top()), p2i(end()));
440   st->print("|TAMS " SHR_PTR_FORMAT,
441             p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
442   st->print("|UWM " SHR_PTR_FORMAT,
443             p2i(_update_watermark));
444   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
445   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
446   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
447   if (ShenandoahHeap::heap()->mode()->is_generational()) {
448     st->print("|P " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()),   proper_unit_for_byte_size(get_plab_allocs()));
449   }
450   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
451   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
452   st->print("|CP " SIZE_FORMAT_W(3), pin_count());
453   st->cr();
454 
455 #undef SHR_PTR_FORMAT
456 }
457 
458 // oop_iterate without closure, return true if completed without cancellation
459 bool ShenandoahHeapRegion::oop_coalesce_and_fill(bool cancellable) {
460 
461   // Consider yielding to cancel/preemption request after this many coalesce operations (skip marked, or coalesce free).
462   const size_t preemption_stride = 128;
463 
464   assert(!is_humongous(), "No need to fill or coalesce humongous regions");
465   if (!is_active()) {
466     end_preemptible_coalesce_and_fill();
467     return true;
468   }
469 
470   ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
471   ShenandoahMarkingContext* marking_context = heap->marking_context();
472 
473   // Expect marking to be completed before these threads invoke this service.
474   assert(heap->active_generation()->is_mark_complete(), "sanity");
475 
476   // All objects above TAMS are considered live even though their mark bits will not be set.  Note that young-
477   // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
478   // while the old-gen concurrent marking is ongoing.  These newly promoted objects will reside above TAMS
479   // and will be treated as live during the current old-gen marking pass, even though they will not be
480   // explicitly marked.
481   HeapWord* t = marking_context->top_at_mark_start(this);
482 
483   // Resume coalesce and fill from this address
484   HeapWord* obj_addr = resume_coalesce_and_fill();
485 
486   size_t ops_before_preempt_check = preemption_stride;
487   while (obj_addr < t) {
488     oop obj = cast_to_oop(obj_addr);
489     if (marking_context->is_marked(obj)) {
490       assert(obj->klass() != nullptr, "klass should not be nullptr");
491       obj_addr += obj->size();
492     } else {
493       // Object is not marked.  Coalesce and fill dead object with dead neighbors.
494       HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
495       assert(next_marked_obj <= t, "next marked object cannot exceed top");
496       size_t fill_size = next_marked_obj - obj_addr;
497       assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
498       ShenandoahHeap::fill_with_object(obj_addr, fill_size);
499       heap->old_generation()->card_scan()->coalesce_objects(obj_addr, fill_size);
500       obj_addr = next_marked_obj;
501     }
502     if (cancellable && ops_before_preempt_check-- == 0) {
503       if (heap->cancelled_gc()) {
504         suspend_coalesce_and_fill(obj_addr);
505         return false;
506       }
507       ops_before_preempt_check = preemption_stride;
508     }
509   }
510   // Mark that this region has been coalesced and filled
511   end_preemptible_coalesce_and_fill();
512   return true;
513 }
514 
515 // DO NOT CANCEL.  If this worker thread has accepted responsibility for scanning a particular range of addresses, it
516 // must finish the work before it can be cancelled.
517 void ShenandoahHeapRegion::oop_iterate_humongous_slice(OopIterateClosure* blk, bool dirty_only,
518                                                        HeapWord* start, size_t words, bool write_table) {
519   assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards");
520   assert(is_humongous(), "only humongous region here");
521   ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
522 
523   // Find head.
524   ShenandoahHeapRegion* r = humongous_start_region();
525   assert(r->is_humongous_start(), "need humongous head here");
526   assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words,
527          "slice must be integral number of cards");
528 
529   oop obj = cast_to_oop(r->bottom());
530   RememberedScanner* scanner = heap->old_generation()->card_scan();
531   size_t card_index = scanner->card_index_for_addr(start);
532   size_t num_cards = words / CardTable::card_size_in_words();
533 
534   if (dirty_only) {
535     if (write_table) {
536       while (num_cards-- > 0) {
537         if (scanner->is_write_card_dirty(card_index++)) {
538           obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
539         }
540         start += CardTable::card_size_in_words();
541       }
542     } else {
543       while (num_cards-- > 0) {
544         if (scanner->is_card_dirty(card_index++)) {
545           obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
546         }
547         start += CardTable::card_size_in_words();
548       }
549     }
550   } else {
551     // Scan all data, regardless of whether cards are dirty
552     obj->oop_iterate(blk, MemRegion(start, start + num_cards * CardTable::card_size_in_words()));
553   }
554 }
555 
556 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
557   ShenandoahHeap* heap = ShenandoahHeap::heap();
558   assert(is_humongous(), "Must be a part of the humongous region");
559   size_t i = index();
560   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
561   while (!r->is_humongous_start()) {
562     assert(i > 0, "Sanity");
563     i--;
564     r = heap->get_region(i);
565     assert(r->is_humongous(), "Must be a part of the humongous region");
566   }
567   assert(r->is_humongous_start(), "Must be");
568   return r;
569 }
570 
571 void ShenandoahHeapRegion::recycle() {
572   shenandoah_assert_heaplocked();
573   ShenandoahHeap* heap = ShenandoahHeap::heap();
574   ShenandoahGeneration* generation = heap->generation_for(affiliation());
575 
576   heap->decrease_used(generation, used());
577   generation->decrement_affiliated_region_count();
578 
579   set_top(bottom());
580   clear_live_data();
581 
582   reset_alloc_metadata();
583 
584   heap->marking_context()->reset_top_at_mark_start(this);
585   set_update_watermark(bottom());
586 
587   make_empty();
588 
589   set_affiliation(FREE);
590   if (ZapUnusedHeapArea) {
591     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
592   }
593 }
594 
595 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
596   assert(MemRegion(bottom(), end()).contains(p),
597          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
598          p2i(p), p2i(bottom()), p2i(end()));
599   if (p >= top()) {
600     return top();
601   } else {
602     HeapWord* last = bottom();
603     HeapWord* cur = last;
604     while (cur <= p) {
605       last = cur;
606       cur += cast_to_oop(cur)->size();
607     }
608     shenandoah_assert_correct(nullptr, cast_to_oop(last));
609     return last;
610   }
611 }
612 
613 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
614   assert(MemRegion(bottom(), end()).contains(p),
615          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
616          p2i(p), p2i(bottom()), p2i(end()));
617   if (p < top()) {
618     return cast_to_oop(p)->size();
619   } else {
620     assert(p == top(), "just checking");
621     return pointer_delta(end(), (HeapWord*) p);
622   }
623 }
624 
625 size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
626   // Absolute minimums we should not ever break.
627   static const size_t MIN_REGION_SIZE = 256*K;
628 
629   if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
630     FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
631   }
632 
633   // Generational Shenandoah needs this alignment for card tables.
634   if (strcmp(ShenandoahGCMode, "generational") == 0) {
635     max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint());
636   }
637 
638   size_t region_size;
639   if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
640     if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
641       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
642                       "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
643                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
644                       MIN_NUM_REGIONS,
645                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
646       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
647     }
648     if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
649       err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
650                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
651                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
652       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
653     }
654     if (ShenandoahMinRegionSize < MinTLABSize) {
655       err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
656                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
657                       byte_size_in_proper_unit(MinTLABSize),             proper_unit_for_byte_size(MinTLABSize));
658       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
659     }
660     if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
661       err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
662                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
663                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
664       vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
665     }
666     if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
667       err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
668                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
669                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
670       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
671     }
672 
673     // We rapidly expand to max_heap_size in most scenarios, so that is the measure
674     // for usual heap sizes. Do not depend on initial_heap_size here.
675     region_size = max_heap_size / ShenandoahTargetNumRegions;
676 
677     // Now make sure that we don't go over or under our limits.
678     region_size = MAX2(ShenandoahMinRegionSize, region_size);
679     region_size = MIN2(ShenandoahMaxRegionSize, region_size);
680 
681   } else {
682     if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
683       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
684                               "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
685                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
686                       MIN_NUM_REGIONS,
687                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
688       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
689     }
690     if (ShenandoahRegionSize < ShenandoahMinRegionSize) {
691       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
692                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
693                       byte_size_in_proper_unit(ShenandoahMinRegionSize),  proper_unit_for_byte_size(ShenandoahMinRegionSize));
694       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
695     }
696     if (ShenandoahRegionSize > ShenandoahMaxRegionSize) {
697       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
698                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
699                       byte_size_in_proper_unit(ShenandoahMaxRegionSize),  proper_unit_for_byte_size(ShenandoahMaxRegionSize));
700       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
701     }
702     region_size = ShenandoahRegionSize;
703   }
704 
705   // Make sure region size and heap size are page aligned.
706   // If large pages are used, we ensure that region size is aligned to large page size if
707   // heap size is large enough to accommodate minimal number of regions. Otherwise, we align
708   // region size to regular page size.
709 
710   // Figure out page size to use, and aligns up heap to page size
711   size_t page_size = os::vm_page_size();
712   if (UseLargePages) {
713     size_t large_page_size = os::large_page_size();
714     max_heap_size = align_up(max_heap_size, large_page_size);
715     if ((max_heap_size / align_up(region_size, large_page_size)) >= MIN_NUM_REGIONS) {
716       page_size = large_page_size;
717     } else {
718       // Should have been checked during argument initialization
719       assert(!ShenandoahUncommit, "Uncommit requires region size aligns to large page size");
720     }
721   } else {
722     max_heap_size = align_up(max_heap_size, page_size);
723   }
724 
725   // Align region size to page size
726   region_size = align_up(region_size, page_size);
727 
728   int region_size_log = log2i(region_size);
729   // Recalculate the region size to make sure it's a power of
730   // 2. This means that region_size is the largest power of 2 that's
731   // <= what we've calculated so far.
732   region_size = size_t(1) << region_size_log;
733 
734   // Now, set up the globals.
735   guarantee(RegionSizeBytesShift == 0, "we should only set it once");
736   RegionSizeBytesShift = (size_t)region_size_log;
737 
738   guarantee(RegionSizeWordsShift == 0, "we should only set it once");
739   RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
740 
741   guarantee(RegionSizeBytes == 0, "we should only set it once");
742   RegionSizeBytes = region_size;
743   RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
744   assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
745 
746   guarantee(RegionSizeWordsMask == 0, "we should only set it once");
747   RegionSizeWordsMask = RegionSizeWords - 1;
748 
749   guarantee(RegionSizeBytesMask == 0, "we should only set it once");
750   RegionSizeBytesMask = RegionSizeBytes - 1;
751 
752   guarantee(RegionCount == 0, "we should only set it once");
753   RegionCount = align_up(max_heap_size, RegionSizeBytes) / RegionSizeBytes;
754   guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
755 
756   guarantee(HumongousThresholdWords == 0, "we should only set it once");
757   HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
758   HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
759   assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
760 
761   guarantee(HumongousThresholdBytes == 0, "we should only set it once");
762   HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
763   assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
764 
765   guarantee(MaxTLABSizeWords == 0, "we should only set it once");
766   MaxTLABSizeWords = MIN2(RegionSizeWords, HumongousThresholdWords);
767   MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
768 
769   guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
770   MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
771   assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
772 
773   return max_heap_size;
774 }
775 
776 void ShenandoahHeapRegion::do_commit() {
777   ShenandoahHeap* heap = ShenandoahHeap::heap();
778   if (!heap->is_heap_region_special() && !os::commit_memory((char *) bottom(), RegionSizeBytes, false)) {
779     report_java_out_of_memory("Unable to commit region");
780   }
781   if (!heap->commit_bitmap_slice(this)) {
782     report_java_out_of_memory("Unable to commit bitmaps for region");
783   }
784   if (AlwaysPreTouch) {
785     os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size());
786   }
787   heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
788 }
789 
790 void ShenandoahHeapRegion::do_uncommit() {
791   ShenandoahHeap* heap = ShenandoahHeap::heap();
792   if (!heap->is_heap_region_special() && !os::uncommit_memory((char *) bottom(), RegionSizeBytes)) {
793     report_java_out_of_memory("Unable to uncommit region");
794   }
795   if (!heap->uncommit_bitmap_slice(this)) {
796     report_java_out_of_memory("Unable to uncommit bitmaps for region");
797   }
798   heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
799 }
800 
801 void ShenandoahHeapRegion::set_state(RegionState to) {
802   EventShenandoahHeapRegionStateChange evt;
803   if (evt.should_commit()){
804     evt.set_index((unsigned) index());
805     evt.set_start((uintptr_t)bottom());
806     evt.set_used(used());
807     evt.set_from(_state);
808     evt.set_to(to);
809     evt.commit();
810   }
811   _state = to;
812 }
813 
814 void ShenandoahHeapRegion::record_pin() {
815   Atomic::add(&_critical_pins, (size_t)1);
816 }
817 
818 void ShenandoahHeapRegion::record_unpin() {
819   assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
820   Atomic::sub(&_critical_pins, (size_t)1);
821 }
822 
823 size_t ShenandoahHeapRegion::pin_count() const {
824   return Atomic::load(&_critical_pins);
825 }
826 
827 void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) {
828   ShenandoahHeap* heap = ShenandoahHeap::heap();
829 
830   ShenandoahAffiliation region_affiliation = heap->region_affiliation(this);
831   {
832     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
833     log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
834                   ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
835                   index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation),
836                   p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this)));
837   }
838 
839 #ifdef ASSERT
840   {
841     // During full gc, heap->complete_marking_context() is not valid, may equal nullptr.
842     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
843     size_t idx = this->index();
844     HeapWord* top_bitmap = ctx->top_bitmap(this);
845 
846     assert(ctx->is_bitmap_clear_range(top_bitmap, _end),
847            "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx,
848            p2i(top_bitmap), p2i(_end));
849   }
850 #endif
851 
852   if (region_affiliation == new_affiliation) {
853     return;
854   }
855 
856   if (!heap->mode()->is_generational()) {
857     log_trace(gc)("Changing affiliation of region %zu from %s to %s",
858                   index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation));
859     heap->set_affiliation(this, new_affiliation);
860     return;
861   }
862 
863   switch (new_affiliation) {
864     case FREE:
865       assert(!has_live(), "Free region should not have live data");
866       break;
867     case YOUNG_GENERATION:
868       reset_age();
869       break;
870     case OLD_GENERATION:
871       // TODO: should we reset_age() for OLD as well?  Examine invocations of set_affiliation(). Some contexts redundantly
872       //       invoke reset_age().
873       break;
874     default:
875       ShouldNotReachHere();
876       return;
877   }
878   heap->set_affiliation(this, new_affiliation);
879 }
880 
881 void ShenandoahHeapRegion::decrement_humongous_waste() const {
882   assert(is_humongous(), "Should only use this for humongous regions");
883   size_t waste_bytes = free();
884   if (waste_bytes > 0) {
885     ShenandoahHeap* heap = ShenandoahHeap::heap();
886     ShenandoahGeneration* generation = heap->generation_for(affiliation());
887     heap->decrease_humongous_waste(generation, waste_bytes);
888   }
889 }