1 /*
  2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shared/space.inline.hpp"
 27 #include "gc/shared/tlab_globals.hpp"
 28 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 31 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 32 #include "jfr/jfrEvents.hpp"
 33 #include "memory/allocation.hpp"
 34 #include "memory/iterator.inline.hpp"
 35 #include "memory/resourceArea.hpp"
 36 #include "memory/universe.hpp"
 37 #include "oops/oop.inline.hpp"
 38 #include "runtime/atomic.hpp"
 39 #include "runtime/globals_extension.hpp"
 40 #include "runtime/java.hpp"
 41 #include "runtime/mutexLocker.hpp"
 42 #include "runtime/os.hpp"
 43 #include "runtime/safepoint.hpp"
 44 #include "utilities/powerOfTwo.hpp"
 45 
 46 size_t ShenandoahHeapRegion::RegionCount = 0;
 47 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
 48 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
 49 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
 50 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
 51 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
 52 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
 53 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
 54 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
 55 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
 56 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
 57 
 58 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
 59   _index(index),
 60   _bottom(start),
 61   _end(start + RegionSizeWords),
 62   _new_top(NULL),
 63   _empty_time(os::elapsedTime()),
 64   _state(committed ? _empty_committed : _empty_uncommitted),
 65   _top(start),
 66   _tlab_allocs(0),
 67   _gclab_allocs(0),
 68   _live_data(0),
 69   _critical_pins(0),
 70   _update_watermark(start) {
 71 
 72   assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
 73          "invalid space boundaries");
 74   if (ZapUnusedHeapArea && committed) {
 75     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
 76   }
 77 }
 78 
 79 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
 80   ResourceMark rm;
 81   stringStream ss;
 82   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
 83   print_on(&ss);
 84   fatal("%s", ss.as_string());
 85 }
 86 
 87 void ShenandoahHeapRegion::make_regular_allocation() {
 88   shenandoah_assert_heaplocked();
 89 
 90   switch (_state) {
 91     case _empty_uncommitted:
 92       do_commit();
 93     case _empty_committed:
 94       set_state(_regular);
 95     case _regular:
 96     case _pinned:
 97       return;
 98     default:
 99       report_illegal_transition("regular allocation");
100   }
101 }
102 
103 void ShenandoahHeapRegion::make_regular_bypass() {
104   shenandoah_assert_heaplocked();
105   assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
106           "only for full or degen GC");
107 
108   switch (_state) {
109     case _empty_uncommitted:
110       do_commit();
111     case _empty_committed:
112     case _cset:
113     case _humongous_start:
114     case _humongous_cont:
115       set_state(_regular);
116       return;
117     case _pinned_cset:
118       set_state(_pinned);
119       return;
120     case _regular:
121     case _pinned:
122       return;
123     default:
124       report_illegal_transition("regular bypass");
125   }
126 }
127 
128 void ShenandoahHeapRegion::make_humongous_start() {
129   shenandoah_assert_heaplocked();
130   switch (_state) {
131     case _empty_uncommitted:
132       do_commit();
133     case _empty_committed:
134       set_state(_humongous_start);
135       return;
136     default:
137       report_illegal_transition("humongous start allocation");
138   }
139 }
140 
141 void ShenandoahHeapRegion::make_humongous_start_bypass() {
142   shenandoah_assert_heaplocked();
143   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
144 
145   switch (_state) {
146     case _empty_committed:
147     case _regular:
148     case _humongous_start:
149     case _humongous_cont:
150       set_state(_humongous_start);
151       return;
152     default:
153       report_illegal_transition("humongous start bypass");
154   }
155 }
156 
157 void ShenandoahHeapRegion::make_humongous_cont() {
158   shenandoah_assert_heaplocked();
159   switch (_state) {
160     case _empty_uncommitted:
161       do_commit();
162     case _empty_committed:
163      set_state(_humongous_cont);
164       return;
165     default:
166       report_illegal_transition("humongous continuation allocation");
167   }
168 }
169 
170 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
171   shenandoah_assert_heaplocked();
172   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
173 
174   switch (_state) {
175     case _empty_committed:
176     case _regular:
177     case _humongous_start:
178     case _humongous_cont:
179       set_state(_humongous_cont);
180       return;
181     default:
182       report_illegal_transition("humongous continuation bypass");
183   }
184 }
185 
186 void ShenandoahHeapRegion::make_pinned() {
187   shenandoah_assert_heaplocked();
188   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
189 
190   switch (_state) {
191     case _regular:
192       set_state(_pinned);
193     case _pinned_cset:
194     case _pinned:
195       return;
196     case _humongous_start:
197       set_state(_pinned_humongous_start);
198     case _pinned_humongous_start:
199       return;
200     case _cset:
201       _state = _pinned_cset;
202       return;
203     default:
204       report_illegal_transition("pinning");
205   }
206 }
207 
208 void ShenandoahHeapRegion::make_unpinned() {
209   shenandoah_assert_heaplocked();
210   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
211 
212   switch (_state) {
213     case _pinned:
214       set_state(_regular);
215       return;
216     case _regular:
217     case _humongous_start:
218       return;
219     case _pinned_cset:
220       set_state(_cset);
221       return;
222     case _pinned_humongous_start:
223       set_state(_humongous_start);
224       return;
225     default:
226       report_illegal_transition("unpinning");
227   }
228 }
229 
230 void ShenandoahHeapRegion::make_cset() {
231   shenandoah_assert_heaplocked();
232   switch (_state) {
233     case _regular:
234       set_state(_cset);
235     case _cset:
236       return;
237     default:
238       report_illegal_transition("cset");
239   }
240 }
241 
242 void ShenandoahHeapRegion::make_trash() {
243   shenandoah_assert_heaplocked();
244   switch (_state) {
245     case _cset:
246       // Reclaiming cset regions
247     case _humongous_start:
248     case _humongous_cont:
249       // Reclaiming humongous regions
250     case _regular:
251       // Immediate region reclaim
252       set_state(_trash);
253       return;
254     default:
255       report_illegal_transition("trashing");
256   }
257 }
258 
259 void ShenandoahHeapRegion::make_trash_immediate() {
260   make_trash();
261 
262   // On this path, we know there are no marked objects in the region,
263   // tell marking context about it to bypass bitmap resets.
264   ShenandoahHeap::heap()->complete_marking_context()->reset_top_bitmap(this);
265 }
266 
267 void ShenandoahHeapRegion::make_empty() {
268   shenandoah_assert_heaplocked();
269   switch (_state) {
270     case _trash:
271       set_state(_empty_committed);
272       _empty_time = os::elapsedTime();
273       return;
274     default:
275       report_illegal_transition("emptying");
276   }
277 }
278 
279 void ShenandoahHeapRegion::make_uncommitted() {
280   shenandoah_assert_heaplocked();
281   switch (_state) {
282     case _empty_committed:
283       do_uncommit();
284       set_state(_empty_uncommitted);
285       return;
286     default:
287       report_illegal_transition("uncommiting");
288   }
289 }
290 
291 void ShenandoahHeapRegion::make_committed_bypass() {
292   shenandoah_assert_heaplocked();
293   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
294 
295   switch (_state) {
296     case _empty_uncommitted:
297       do_commit();
298       set_state(_empty_committed);
299       return;
300     default:
301       report_illegal_transition("commit bypass");
302   }
303 }
304 
305 void ShenandoahHeapRegion::reset_alloc_metadata() {
306   _tlab_allocs = 0;
307   _gclab_allocs = 0;
308 }
309 
310 size_t ShenandoahHeapRegion::get_shared_allocs() const {
311   return used() - (_tlab_allocs + _gclab_allocs) * HeapWordSize;
312 }
313 
314 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
315   return _tlab_allocs * HeapWordSize;
316 }
317 
318 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
319   return _gclab_allocs * HeapWordSize;
320 }
321 
322 void ShenandoahHeapRegion::set_live_data(size_t s) {
323   assert(Thread::current()->is_VM_thread(), "by VM thread");
324   _live_data = (s >> LogHeapWordSize);
325 }
326 
327 void ShenandoahHeapRegion::print_on(outputStream* st) const {
328   st->print("|");
329   st->print(SIZE_FORMAT_W(5), this->_index);
330 
331   switch (_state) {
332     case _empty_uncommitted:
333       st->print("|EU ");
334       break;
335     case _empty_committed:
336       st->print("|EC ");
337       break;
338     case _regular:
339       st->print("|R  ");
340       break;
341     case _humongous_start:
342       st->print("|H  ");
343       break;
344     case _pinned_humongous_start:
345       st->print("|HP ");
346       break;
347     case _humongous_cont:
348       st->print("|HC ");
349       break;
350     case _cset:
351       st->print("|CS ");
352       break;
353     case _trash:
354       st->print("|T  ");
355       break;
356     case _pinned:
357       st->print("|P  ");
358       break;
359     case _pinned_cset:
360       st->print("|CSP");
361       break;
362     default:
363       ShouldNotReachHere();
364   }
365   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
366             p2i(bottom()), p2i(top()), p2i(end()));
367   st->print("|TAMS " INTPTR_FORMAT_W(12),
368             p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
369   st->print("|UWM " INTPTR_FORMAT_W(12),
370             p2i(_update_watermark));
371   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
372   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
373   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
374   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
375   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
376   st->print("|CP " SIZE_FORMAT_W(3), pin_count());
377   st->cr();
378 }
379 
380 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
381   if (!is_active()) return;
382   if (is_humongous()) {
383     oop_iterate_humongous(blk);
384   } else {
385     oop_iterate_objects(blk);
386   }
387 }
388 
389 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
390   assert(! is_humongous(), "no humongous region here");
391   HeapWord* obj_addr = bottom();
392   HeapWord* t = top();
393   // Could call objects iterate, but this is easier.
394   while (obj_addr < t) {
395     oop obj = cast_to_oop(obj_addr);
396     obj_addr += obj->oop_iterate_size(blk);
397   }
398 }
399 
400 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
401   assert(is_humongous(), "only humongous region here");
402   // Find head.
403   ShenandoahHeapRegion* r = humongous_start_region();
404   assert(r->is_humongous_start(), "need humongous head here");
405   oop obj = cast_to_oop(r->bottom());
406   obj->oop_iterate(blk, MemRegion(bottom(), top()));
407 }
408 
409 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
410   ShenandoahHeap* heap = ShenandoahHeap::heap();
411   assert(is_humongous(), "Must be a part of the humongous region");
412   size_t i = index();
413   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
414   while (!r->is_humongous_start()) {
415     assert(i > 0, "Sanity");
416     i--;
417     r = heap->get_region(i);
418     assert(r->is_humongous(), "Must be a part of the humongous region");
419   }
420   assert(r->is_humongous_start(), "Must be");
421   return r;
422 }
423 
424 void ShenandoahHeapRegion::recycle() {
425   set_top(bottom());
426   clear_live_data();
427 
428   reset_alloc_metadata();
429 
430   ShenandoahHeap::heap()->marking_context()->reset_top_at_mark_start(this);
431   set_update_watermark(bottom());
432 
433   make_empty();
434 
435   if (ZapUnusedHeapArea) {
436     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
437   }
438 }
439 
440 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
441   assert(MemRegion(bottom(), end()).contains(p),
442          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
443          p2i(p), p2i(bottom()), p2i(end()));
444   if (p >= top()) {
445     return top();
446   } else {
447     HeapWord* last = bottom();
448     HeapWord* cur = last;
449     while (cur <= p) {
450       last = cur;
451       cur += cast_to_oop(cur)->size();
452     }
453     shenandoah_assert_correct(NULL, cast_to_oop(last));
454     return last;
455   }
456 }
457 
458 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
459   assert(MemRegion(bottom(), end()).contains(p),
460          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
461          p2i(p), p2i(bottom()), p2i(end()));
462   if (p < top()) {
463     return cast_to_oop(p)->size();
464   } else {
465     assert(p == top(), "just checking");
466     return pointer_delta(end(), (HeapWord*) p);
467   }
468 }
469 
470 size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
471   // Absolute minimums we should not ever break.
472   static const size_t MIN_REGION_SIZE = 256*K;
473 
474   if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
475     FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
476   }
477 
478   size_t region_size;
479   if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
480     if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
481       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
482                       "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
483                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
484                       MIN_NUM_REGIONS,
485                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
486       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
487     }
488     if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
489       err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
490                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
491                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
492       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
493     }
494     if (ShenandoahMinRegionSize < MinTLABSize) {
495       err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
496                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
497                       byte_size_in_proper_unit(MinTLABSize),             proper_unit_for_byte_size(MinTLABSize));
498       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
499     }
500     if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
501       err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
502                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
503                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
504       vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
505     }
506     if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
507       err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
508                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
509                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
510       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
511     }
512 
513     // We rapidly expand to max_heap_size in most scenarios, so that is the measure
514     // for usual heap sizes. Do not depend on initial_heap_size here.
515     region_size = max_heap_size / ShenandoahTargetNumRegions;
516 
517     // Now make sure that we don't go over or under our limits.
518     region_size = MAX2(ShenandoahMinRegionSize, region_size);
519     region_size = MIN2(ShenandoahMaxRegionSize, region_size);
520 
521   } else {
522     if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
523       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
524                               "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
525                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
526                       MIN_NUM_REGIONS,
527                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
528       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
529     }
530     if (ShenandoahRegionSize < ShenandoahMinRegionSize) {
531       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
532                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
533                       byte_size_in_proper_unit(ShenandoahMinRegionSize),  proper_unit_for_byte_size(ShenandoahMinRegionSize));
534       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
535     }
536     if (ShenandoahRegionSize > ShenandoahMaxRegionSize) {
537       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
538                       byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
539                       byte_size_in_proper_unit(ShenandoahMaxRegionSize),  proper_unit_for_byte_size(ShenandoahMaxRegionSize));
540       vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
541     }
542     region_size = ShenandoahRegionSize;
543   }
544 
545   // Make sure region size and heap size are page aligned.
546   // If large pages are used, we ensure that region size is aligned to large page size if
547   // heap size is large enough to accommodate minimal number of regions. Otherwise, we align
548   // region size to regular page size.
549 
550   // Figure out page size to use, and aligns up heap to page size
551   int page_size = os::vm_page_size();
552   if (UseLargePages) {
553     size_t large_page_size = os::large_page_size();
554     max_heap_size = align_up(max_heap_size, large_page_size);
555     if ((max_heap_size / align_up(region_size, large_page_size)) >= MIN_NUM_REGIONS) {
556       page_size = (int)large_page_size;
557     } else {
558       // Should have been checked during argument initialization
559       assert(!ShenandoahUncommit, "Uncommit requires region size aligns to large page size");
560     }
561   } else {
562     max_heap_size = align_up(max_heap_size, page_size);
563   }
564 
565   // Align region size to page size
566   region_size = align_up(region_size, page_size);
567 
568   int region_size_log = log2i(region_size);
569   // Recalculate the region size to make sure it's a power of
570   // 2. This means that region_size is the largest power of 2 that's
571   // <= what we've calculated so far.
572   region_size = size_t(1) << region_size_log;
573 
574   // Now, set up the globals.
575   guarantee(RegionSizeBytesShift == 0, "we should only set it once");
576   RegionSizeBytesShift = (size_t)region_size_log;
577 
578   guarantee(RegionSizeWordsShift == 0, "we should only set it once");
579   RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
580 
581   guarantee(RegionSizeBytes == 0, "we should only set it once");
582   RegionSizeBytes = region_size;
583   RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
584   assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
585 
586   guarantee(RegionSizeWordsMask == 0, "we should only set it once");
587   RegionSizeWordsMask = RegionSizeWords - 1;
588 
589   guarantee(RegionSizeBytesMask == 0, "we should only set it once");
590   RegionSizeBytesMask = RegionSizeBytes - 1;
591 
592   guarantee(RegionCount == 0, "we should only set it once");
593   RegionCount = align_up(max_heap_size, RegionSizeBytes) / RegionSizeBytes;
594   guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
595 
596   guarantee(HumongousThresholdWords == 0, "we should only set it once");
597   HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
598   HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
599   assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
600 
601   guarantee(HumongousThresholdBytes == 0, "we should only set it once");
602   HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
603   assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
604 
605   // The rationale for trimming the TLAB sizes has to do with the raciness in
606   // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
607   // about next free size, gets the answer for region #N, goes away for a while, then
608   // tries to allocate in region #N, and fail because some other thread have claimed part
609   // of the region #N, and then the freeset allocation code has to retire the region #N,
610   // before moving the allocation to region #N+1.
611   //
612   // The worst case realizes when "answer" is "region size", which means it could
613   // prematurely retire an entire region. Having smaller TLABs does not fix that
614   // completely, but reduces the probability of too wasteful region retirement.
615   // With current divisor, we will waste no more than 1/8 of region size in the worst
616   // case. This also has a secondary effect on collection set selection: even under
617   // the race, the regions would be at least 7/8 used, which allows relying on
618   // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
619   // below the garbage threshold that would never be considered for collection.
620   //
621   // The whole thing is mitigated if Elastic TLABs are enabled.
622   //
623   guarantee(MaxTLABSizeWords == 0, "we should only set it once");
624   MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
625   MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
626 
627   guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
628   MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
629   assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
630 
631   return max_heap_size;
632 }
633 
634 void ShenandoahHeapRegion::do_commit() {
635   ShenandoahHeap* heap = ShenandoahHeap::heap();
636   if (!heap->is_heap_region_special() && !os::commit_memory((char *) bottom(), RegionSizeBytes, false)) {
637     report_java_out_of_memory("Unable to commit region");
638   }
639   if (!heap->commit_bitmap_slice(this)) {
640     report_java_out_of_memory("Unable to commit bitmaps for region");
641   }
642   if (AlwaysPreTouch) {
643     os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size());
644   }
645   heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
646 }
647 
648 void ShenandoahHeapRegion::do_uncommit() {
649   ShenandoahHeap* heap = ShenandoahHeap::heap();
650   if (!heap->is_heap_region_special() && !os::uncommit_memory((char *) bottom(), RegionSizeBytes)) {
651     report_java_out_of_memory("Unable to uncommit region");
652   }
653   if (!heap->uncommit_bitmap_slice(this)) {
654     report_java_out_of_memory("Unable to uncommit bitmaps for region");
655   }
656   heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
657 }
658 
659 void ShenandoahHeapRegion::set_state(RegionState to) {
660   EventShenandoahHeapRegionStateChange evt;
661   if (evt.should_commit()){
662     evt.set_index((unsigned) index());
663     evt.set_start((uintptr_t)bottom());
664     evt.set_used(used());
665     evt.set_from(_state);
666     evt.set_to(to);
667     evt.commit();
668   }
669   _state = to;
670 }
671 
672 void ShenandoahHeapRegion::record_pin() {
673   Atomic::add(&_critical_pins, (size_t)1);
674 }
675 
676 void ShenandoahHeapRegion::record_unpin() {
677   assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
678   Atomic::sub(&_critical_pins, (size_t)1);
679 }
680 
681 size_t ShenandoahHeapRegion::pin_count() const {
682   return Atomic::load(&_critical_pins);
683 }