< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

Print this page

   1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/space.inline.hpp"
  27 #include "gc/shared/tlab_globals.hpp"

  28 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  31 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"




  32 #include "jfr/jfrEvents.hpp"
  33 #include "memory/allocation.hpp"
  34 #include "memory/iterator.inline.hpp"
  35 #include "memory/resourceArea.hpp"
  36 #include "memory/universe.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "runtime/atomic.hpp"
  39 #include "runtime/globals_extension.hpp"
  40 #include "runtime/java.hpp"
  41 #include "runtime/mutexLocker.hpp"
  42 #include "runtime/os.hpp"
  43 #include "runtime/safepoint.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 

  46 size_t ShenandoahHeapRegion::RegionCount = 0;
  47 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
  48 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
  49 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
  50 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
  51 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
  52 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
  53 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
  54 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
  55 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
  56 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
  57 
  58 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
  59   _index(index),
  60   _bottom(start),
  61   _end(start + RegionSizeWords),
  62   _new_top(NULL),
  63   _empty_time(os::elapsedTime()),
  64   _state(committed ? _empty_committed : _empty_uncommitted),
  65   _top(start),
  66   _tlab_allocs(0),
  67   _gclab_allocs(0),


  68   _live_data(0),
  69   _critical_pins(0),
  70   _update_watermark(start) {


  71 
  72   assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
  73          "invalid space boundaries");
  74   if (ZapUnusedHeapArea && committed) {
  75     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
  76   }
  77 }
  78 
  79 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
  80   ResourceMark rm;
  81   stringStream ss;
  82   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
  83   print_on(&ss);
  84   fatal("%s", ss.as_string());
  85 }
  86 
  87 void ShenandoahHeapRegion::make_regular_allocation() {
  88   shenandoah_assert_heaplocked();
  89 
  90   switch (_state) {
  91     case _empty_uncommitted:
  92       do_commit();
  93     case _empty_committed:

  94       set_state(_regular);
  95     case _regular:
  96     case _pinned:
  97       return;
  98     default:
  99       report_illegal_transition("regular allocation");
 100   }
 101 }
 102 




















 103 void ShenandoahHeapRegion::make_regular_bypass() {
 104   shenandoah_assert_heaplocked();
 105   assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
 106           "only for full or degen GC");
 107 
 108   switch (_state) {
 109     case _empty_uncommitted:
 110       do_commit();
 111     case _empty_committed:
 112     case _cset:
 113     case _humongous_start:
 114     case _humongous_cont:
 115       set_state(_regular);
 116       return;
 117     case _pinned_cset:
 118       set_state(_pinned);
 119       return;
 120     case _regular:
 121     case _pinned:
 122       return;
 123     default:
 124       report_illegal_transition("regular bypass");
 125   }
 126 }
 127 
 128 void ShenandoahHeapRegion::make_humongous_start() {
 129   shenandoah_assert_heaplocked();

 130   switch (_state) {
 131     case _empty_uncommitted:
 132       do_commit();
 133     case _empty_committed:
 134       set_state(_humongous_start);
 135       return;
 136     default:
 137       report_illegal_transition("humongous start allocation");
 138   }
 139 }
 140 
 141 void ShenandoahHeapRegion::make_humongous_start_bypass() {
 142   shenandoah_assert_heaplocked();
 143   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 144 

 145   switch (_state) {
 146     case _empty_committed:
 147     case _regular:
 148     case _humongous_start:
 149     case _humongous_cont:
 150       set_state(_humongous_start);
 151       return;
 152     default:
 153       report_illegal_transition("humongous start bypass");
 154   }
 155 }
 156 
 157 void ShenandoahHeapRegion::make_humongous_cont() {
 158   shenandoah_assert_heaplocked();

 159   switch (_state) {
 160     case _empty_uncommitted:
 161       do_commit();
 162     case _empty_committed:
 163      set_state(_humongous_cont);
 164       return;
 165     default:
 166       report_illegal_transition("humongous continuation allocation");
 167   }
 168 }
 169 
 170 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
 171   shenandoah_assert_heaplocked();
 172   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 173 

 174   switch (_state) {
 175     case _empty_committed:
 176     case _regular:
 177     case _humongous_start:
 178     case _humongous_cont:
 179       set_state(_humongous_cont);
 180       return;
 181     default:
 182       report_illegal_transition("humongous continuation bypass");
 183   }
 184 }
 185 
 186 void ShenandoahHeapRegion::make_pinned() {
 187   shenandoah_assert_heaplocked();
 188   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
 189 
 190   switch (_state) {
 191     case _regular:
 192       set_state(_pinned);
 193     case _pinned_cset:
 194     case _pinned:
 195       return;
 196     case _humongous_start:
 197       set_state(_pinned_humongous_start);
 198     case _pinned_humongous_start:
 199       return;
 200     case _cset:
 201       _state = _pinned_cset;
 202       return;
 203     default:
 204       report_illegal_transition("pinning");
 205   }
 206 }
 207 
 208 void ShenandoahHeapRegion::make_unpinned() {
 209   shenandoah_assert_heaplocked();
 210   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
 211 
 212   switch (_state) {
 213     case _pinned:

 214       set_state(_regular);
 215       return;
 216     case _regular:
 217     case _humongous_start:
 218       return;
 219     case _pinned_cset:
 220       set_state(_cset);
 221       return;
 222     case _pinned_humongous_start:
 223       set_state(_humongous_start);
 224       return;
 225     default:
 226       report_illegal_transition("unpinning");
 227   }
 228 }
 229 
 230 void ShenandoahHeapRegion::make_cset() {
 231   shenandoah_assert_heaplocked();

 232   switch (_state) {
 233     case _regular:
 234       set_state(_cset);
 235     case _cset:
 236       return;
 237     default:
 238       report_illegal_transition("cset");
 239   }
 240 }
 241 
 242 void ShenandoahHeapRegion::make_trash() {
 243   shenandoah_assert_heaplocked();

 244   switch (_state) {
 245     case _cset:
 246       // Reclaiming cset regions
 247     case _humongous_start:
 248     case _humongous_cont:
 249       // Reclaiming humongous regions
 250     case _regular:
 251       // Immediate region reclaim
 252       set_state(_trash);
 253       return;
 254     default:
 255       report_illegal_transition("trashing");
 256   }
 257 }
 258 
 259 void ShenandoahHeapRegion::make_trash_immediate() {
 260   make_trash();
 261 
 262   // On this path, we know there are no marked objects in the region,
 263   // tell marking context about it to bypass bitmap resets.
 264   ShenandoahHeap::heap()->complete_marking_context()->reset_top_bitmap(this);

 265 }
 266 
 267 void ShenandoahHeapRegion::make_empty() {
 268   shenandoah_assert_heaplocked();

 269   switch (_state) {
 270     case _trash:
 271       set_state(_empty_committed);
 272       _empty_time = os::elapsedTime();
 273       return;
 274     default:
 275       report_illegal_transition("emptying");
 276   }
 277 }
 278 
 279 void ShenandoahHeapRegion::make_uncommitted() {
 280   shenandoah_assert_heaplocked();
 281   switch (_state) {
 282     case _empty_committed:
 283       do_uncommit();
 284       set_state(_empty_uncommitted);
 285       return;
 286     default:
 287       report_illegal_transition("uncommiting");
 288   }
 289 }
 290 
 291 void ShenandoahHeapRegion::make_committed_bypass() {
 292   shenandoah_assert_heaplocked();
 293   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 294 
 295   switch (_state) {
 296     case _empty_uncommitted:
 297       do_commit();
 298       set_state(_empty_committed);
 299       return;
 300     default:
 301       report_illegal_transition("commit bypass");
 302   }
 303 }
 304 
 305 void ShenandoahHeapRegion::reset_alloc_metadata() {
 306   _tlab_allocs = 0;
 307   _gclab_allocs = 0;

 308 }
 309 
 310 size_t ShenandoahHeapRegion::get_shared_allocs() const {
 311   return used() - (_tlab_allocs + _gclab_allocs) * HeapWordSize;
 312 }
 313 
 314 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
 315   return _tlab_allocs * HeapWordSize;
 316 }
 317 
 318 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
 319   return _gclab_allocs * HeapWordSize;
 320 }
 321 




 322 void ShenandoahHeapRegion::set_live_data(size_t s) {
 323   assert(Thread::current()->is_VM_thread(), "by VM thread");
 324   _live_data = (s >> LogHeapWordSize);
 325 }
 326 
 327 void ShenandoahHeapRegion::print_on(outputStream* st) const {
 328   st->print("|");
 329   st->print(SIZE_FORMAT_W(5), this->_index);
 330 
 331   switch (_state) {
 332     case _empty_uncommitted:
 333       st->print("|EU ");
 334       break;
 335     case _empty_committed:
 336       st->print("|EC ");
 337       break;
 338     case _regular:
 339       st->print("|R  ");
 340       break;
 341     case _humongous_start:

 345       st->print("|HP ");
 346       break;
 347     case _humongous_cont:
 348       st->print("|HC ");
 349       break;
 350     case _cset:
 351       st->print("|CS ");
 352       break;
 353     case _trash:
 354       st->print("|T  ");
 355       break;
 356     case _pinned:
 357       st->print("|P  ");
 358       break;
 359     case _pinned_cset:
 360       st->print("|CSP");
 361       break;
 362     default:
 363       ShouldNotReachHere();
 364   }













 365   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
 366             p2i(bottom()), p2i(top()), p2i(end()));
 367   st->print("|TAMS " INTPTR_FORMAT_W(12),
 368             p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
 369   st->print("|UWM " INTPTR_FORMAT_W(12),
 370             p2i(_update_watermark));
 371   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
 372   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
 373   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));



 374   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
 375   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
 376   st->print("|CP " SIZE_FORMAT_W(3), pin_count());
 377   st->cr();
 378 }
 379 
 380 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {





























































































 381   if (!is_active()) return;
 382   if (is_humongous()) {


 383     oop_iterate_humongous(blk);
 384   } else {
 385     oop_iterate_objects(blk);
 386   }
 387 }
 388 
 389 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
 390   assert(! is_humongous(), "no humongous region here");
 391   HeapWord* obj_addr = bottom();
 392   HeapWord* t = top();
 393   // Could call objects iterate, but this is easier.




























 394   while (obj_addr < t) {
 395     oop obj = cast_to_oop(obj_addr);
 396     obj_addr += obj->oop_iterate_size(blk);
 397   }
 398 }
 399 


















































 400 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
 401   assert(is_humongous(), "only humongous region here");
 402   // Find head.
 403   ShenandoahHeapRegion* r = humongous_start_region();
 404   assert(r->is_humongous_start(), "need humongous head here");
 405   oop obj = cast_to_oop(r->bottom());
 406   obj->oop_iterate(blk, MemRegion(bottom(), top()));
 407 }
 408 
 409 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
 410   ShenandoahHeap* heap = ShenandoahHeap::heap();
 411   assert(is_humongous(), "Must be a part of the humongous region");
 412   size_t i = index();
 413   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
 414   while (!r->is_humongous_start()) {
 415     assert(i > 0, "Sanity");
 416     i--;
 417     r = heap->get_region(i);
 418     assert(r->is_humongous(), "Must be a part of the humongous region");
 419   }
 420   assert(r->is_humongous_start(), "Must be");
 421   return r;
 422 }
 423 
 424 void ShenandoahHeapRegion::recycle() {








 425   set_top(bottom());
 426   clear_live_data();
 427 
 428   reset_alloc_metadata();
 429 
 430   ShenandoahHeap::heap()->marking_context()->reset_top_at_mark_start(this);
 431   set_update_watermark(bottom());
 432 
 433   make_empty();

 434 
 435   if (ZapUnusedHeapArea) {
 436     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
 437   }
 438 }
 439 
 440 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
 441   assert(MemRegion(bottom(), end()).contains(p),
 442          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
 443          p2i(p), p2i(bottom()), p2i(end()));
 444   if (p >= top()) {
 445     return top();
 446   } else {
 447     HeapWord* last = bottom();
 448     HeapWord* cur = last;
 449     while (cur <= p) {
 450       last = cur;
 451       cur += cast_to_oop(cur)->size();
 452     }
 453     shenandoah_assert_correct(NULL, cast_to_oop(last));

 664     evt.set_used(used());
 665     evt.set_from(_state);
 666     evt.set_to(to);
 667     evt.commit();
 668   }
 669   _state = to;
 670 }
 671 
 672 void ShenandoahHeapRegion::record_pin() {
 673   Atomic::add(&_critical_pins, (size_t)1);
 674 }
 675 
 676 void ShenandoahHeapRegion::record_unpin() {
 677   assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
 678   Atomic::sub(&_critical_pins, (size_t)1);
 679 }
 680 
 681 size_t ShenandoahHeapRegion::pin_count() const {
 682   return Atomic::load(&_critical_pins);
 683 }















































































































   1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/space.inline.hpp"
  27 #include "gc/shared/tlab_globals.hpp"
  28 #include "gc/shenandoah/shenandoahCardTable.hpp"
  29 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  33 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  34 #include "gc/shenandoah/shenandoahGeneration.hpp"
  35 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  36 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  37 #include "jfr/jfrEvents.hpp"
  38 #include "memory/allocation.hpp"
  39 #include "memory/iterator.inline.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "memory/universe.hpp"
  42 #include "oops/oop.inline.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/globals_extension.hpp"
  45 #include "runtime/java.hpp"
  46 #include "runtime/mutexLocker.hpp"
  47 #include "runtime/os.hpp"
  48 #include "runtime/safepoint.hpp"
  49 #include "utilities/powerOfTwo.hpp"
  50 
  51 
  52 size_t ShenandoahHeapRegion::RegionCount = 0;
  53 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
  54 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
  55 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
  56 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
  57 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
  58 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
  59 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
  60 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
  61 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
  62 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
  63 
  64 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
  65   _index(index),
  66   _bottom(start),
  67   _end(start + RegionSizeWords),
  68   _new_top(NULL),
  69   _empty_time(os::elapsedTime()),
  70   _state(committed ? _empty_committed : _empty_uncommitted),
  71   _top(start),
  72   _tlab_allocs(0),
  73   _gclab_allocs(0),
  74   _plab_allocs(0),
  75   _has_young_lab(false),
  76   _live_data(0),
  77   _critical_pins(0),
  78   _update_watermark(start),
  79   _affiliation(FREE),
  80   _age(0) {
  81 
  82   assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
  83          "invalid space boundaries");
  84   if (ZapUnusedHeapArea && committed) {
  85     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
  86   }
  87 }
  88 
  89 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
  90   ResourceMark rm;
  91   stringStream ss;
  92   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
  93   print_on(&ss);
  94   fatal("%s", ss.as_string());
  95 }
  96 
  97 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahRegionAffiliation affiliation) {
  98   shenandoah_assert_heaplocked();
  99   reset_age();
 100   switch (_state) {
 101     case _empty_uncommitted:
 102       do_commit();
 103     case _empty_committed:
 104       set_affiliation(affiliation);
 105       set_state(_regular);
 106     case _regular:
 107     case _pinned:
 108       return;
 109     default:
 110       report_illegal_transition("regular allocation");
 111   }
 112 }
 113 
 114 // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned.  This implements
 115 // behavior previously performed as a side effect of make_regular_bypass().
 116 void ShenandoahHeapRegion::make_young_maybe() {
 117  switch (_state) {
 118    case _empty_uncommitted:
 119    case _empty_committed:
 120    case _cset:
 121    case _humongous_start:
 122    case _humongous_cont:
 123      set_affiliation(YOUNG_GENERATION);
 124      return;
 125    case _pinned_cset:
 126    case _regular:
 127    case _pinned:
 128      return;
 129    default:
 130      assert(false, "Unexpected _state in make_young_maybe");
 131   }
 132 }
 133 
 134 void ShenandoahHeapRegion::make_regular_bypass() {
 135   shenandoah_assert_heaplocked();
 136   assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
 137           "only for full or degen GC");
 138   reset_age();
 139   switch (_state) {
 140     case _empty_uncommitted:
 141       do_commit();
 142     case _empty_committed:
 143     case _cset:
 144     case _humongous_start:
 145     case _humongous_cont:
 146       set_state(_regular);
 147       return;
 148     case _pinned_cset:
 149       set_state(_pinned);
 150       return;
 151     case _regular:
 152     case _pinned:
 153       return;
 154     default:
 155       report_illegal_transition("regular bypass");
 156   }
 157 }
 158 
 159 void ShenandoahHeapRegion::make_humongous_start() {
 160   shenandoah_assert_heaplocked();
 161   reset_age();
 162   switch (_state) {
 163     case _empty_uncommitted:
 164       do_commit();
 165     case _empty_committed:
 166       set_state(_humongous_start);
 167       return;
 168     default:
 169       report_illegal_transition("humongous start allocation");
 170   }
 171 }
 172 
 173 void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahRegionAffiliation affiliation) {
 174   shenandoah_assert_heaplocked();
 175   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 176   set_affiliation(affiliation);
 177   reset_age();
 178   switch (_state) {
 179     case _empty_committed:
 180     case _regular:
 181     case _humongous_start:
 182     case _humongous_cont:
 183       set_state(_humongous_start);
 184       return;
 185     default:
 186       report_illegal_transition("humongous start bypass");
 187   }
 188 }
 189 
 190 void ShenandoahHeapRegion::make_humongous_cont() {
 191   shenandoah_assert_heaplocked();
 192   reset_age();
 193   switch (_state) {
 194     case _empty_uncommitted:
 195       do_commit();
 196     case _empty_committed:
 197      set_state(_humongous_cont);
 198       return;
 199     default:
 200       report_illegal_transition("humongous continuation allocation");
 201   }
 202 }
 203 
 204 void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahRegionAffiliation affiliation) {
 205   shenandoah_assert_heaplocked();
 206   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 207   set_affiliation(affiliation);
 208   reset_age();
 209   switch (_state) {
 210     case _empty_committed:
 211     case _regular:
 212     case _humongous_start:
 213     case _humongous_cont:
 214       set_state(_humongous_cont);
 215       return;
 216     default:
 217       report_illegal_transition("humongous continuation bypass");
 218   }
 219 }
 220 
 221 void ShenandoahHeapRegion::make_pinned() {
 222   shenandoah_assert_heaplocked();
 223   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
 224 
 225   switch (_state) {
 226     case _regular:
 227       set_state(_pinned);
 228     case _pinned_cset:
 229     case _pinned:
 230       return;
 231     case _humongous_start:
 232       set_state(_pinned_humongous_start);
 233     case _pinned_humongous_start:
 234       return;
 235     case _cset:
 236       _state = _pinned_cset;
 237       return;
 238     default:
 239       report_illegal_transition("pinning");
 240   }
 241 }
 242 
 243 void ShenandoahHeapRegion::make_unpinned() {
 244   shenandoah_assert_heaplocked();
 245   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
 246 
 247   switch (_state) {
 248     case _pinned:
 249       assert(affiliation() != FREE, "Pinned region should not be FREE");
 250       set_state(_regular);
 251       return;
 252     case _regular:
 253     case _humongous_start:
 254       return;
 255     case _pinned_cset:
 256       set_state(_cset);
 257       return;
 258     case _pinned_humongous_start:
 259       set_state(_humongous_start);
 260       return;
 261     default:
 262       report_illegal_transition("unpinning");
 263   }
 264 }
 265 
 266 void ShenandoahHeapRegion::make_cset() {
 267   shenandoah_assert_heaplocked();
 268   // Leave age untouched.  We need to consult the age when we are deciding whether to promote evacuated objects.
 269   switch (_state) {
 270     case _regular:
 271       set_state(_cset);
 272     case _cset:
 273       return;
 274     default:
 275       report_illegal_transition("cset");
 276   }
 277 }
 278 
 279 void ShenandoahHeapRegion::make_trash() {
 280   shenandoah_assert_heaplocked();
 281   reset_age();
 282   switch (_state) {
 283     case _cset:
 284       // Reclaiming cset regions
 285     case _humongous_start:
 286     case _humongous_cont:
 287       // Reclaiming humongous regions
 288     case _regular:
 289       // Immediate region reclaim
 290       set_state(_trash);
 291       return;
 292     default:
 293       report_illegal_transition("trashing");
 294   }
 295 }
 296 
 297 void ShenandoahHeapRegion::make_trash_immediate() {
 298   make_trash();
 299 
 300   // On this path, we know there are no marked objects in the region,
 301   // tell marking context about it to bypass bitmap resets.
 302   assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here.");
 303   ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this);
 304 }
 305 
 306 void ShenandoahHeapRegion::make_empty() {
 307   shenandoah_assert_heaplocked();
 308   reset_age();
 309   switch (_state) {
 310     case _trash:
 311       set_state(_empty_committed);
 312       _empty_time = os::elapsedTime();
 313       return;
 314     default:
 315       report_illegal_transition("emptying");
 316   }
 317 }
 318 
 319 void ShenandoahHeapRegion::make_uncommitted() {
 320   shenandoah_assert_heaplocked();
 321   switch (_state) {
 322     case _empty_committed:
 323       do_uncommit();
 324       set_state(_empty_uncommitted);
 325       return;
 326     default:
 327       report_illegal_transition("uncommiting");
 328   }
 329 }
 330 
 331 void ShenandoahHeapRegion::make_committed_bypass() {
 332   shenandoah_assert_heaplocked();
 333   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 334 
 335   switch (_state) {
 336     case _empty_uncommitted:
 337       do_commit();
 338       set_state(_empty_committed);
 339       return;
 340     default:
 341       report_illegal_transition("commit bypass");
 342   }
 343 }
 344 
 345 void ShenandoahHeapRegion::reset_alloc_metadata() {
 346   _tlab_allocs = 0;
 347   _gclab_allocs = 0;
 348   _plab_allocs = 0;
 349 }
 350 
 351 size_t ShenandoahHeapRegion::get_shared_allocs() const {
 352   return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize;
 353 }
 354 
 355 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
 356   return _tlab_allocs * HeapWordSize;
 357 }
 358 
 359 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
 360   return _gclab_allocs * HeapWordSize;
 361 }
 362 
 363 size_t ShenandoahHeapRegion::get_plab_allocs() const {
 364   return _plab_allocs * HeapWordSize;
 365 }
 366 
 367 void ShenandoahHeapRegion::set_live_data(size_t s) {
 368   assert(Thread::current()->is_VM_thread(), "by VM thread");
 369   _live_data = (s >> LogHeapWordSize);
 370 }
 371 
 372 void ShenandoahHeapRegion::print_on(outputStream* st) const {
 373   st->print("|");
 374   st->print(SIZE_FORMAT_W(5), this->_index);
 375 
 376   switch (_state) {
 377     case _empty_uncommitted:
 378       st->print("|EU ");
 379       break;
 380     case _empty_committed:
 381       st->print("|EC ");
 382       break;
 383     case _regular:
 384       st->print("|R  ");
 385       break;
 386     case _humongous_start:

 390       st->print("|HP ");
 391       break;
 392     case _humongous_cont:
 393       st->print("|HC ");
 394       break;
 395     case _cset:
 396       st->print("|CS ");
 397       break;
 398     case _trash:
 399       st->print("|T  ");
 400       break;
 401     case _pinned:
 402       st->print("|P  ");
 403       break;
 404     case _pinned_cset:
 405       st->print("|CSP");
 406       break;
 407     default:
 408       ShouldNotReachHere();
 409   }
 410   switch (_affiliation) {
 411     case ShenandoahRegionAffiliation::FREE:
 412       st->print("|F");
 413       break;
 414     case ShenandoahRegionAffiliation::YOUNG_GENERATION:
 415       st->print("|Y");
 416       break;
 417     case ShenandoahRegionAffiliation::OLD_GENERATION:
 418       st->print("|O");
 419       break;
 420     default:
 421       ShouldNotReachHere();
 422   }
 423   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
 424             p2i(bottom()), p2i(top()), p2i(end()));
 425   st->print("|TAMS " INTPTR_FORMAT_W(12),
 426             p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
 427   st->print("|UWM " INTPTR_FORMAT_W(12),
 428             p2i(_update_watermark));
 429   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
 430   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
 431   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
 432   if (ShenandoahHeap::heap()->mode()->is_generational()) {
 433     st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()),   proper_unit_for_byte_size(get_plab_allocs()));
 434   }
 435   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
 436   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
 437   st->print("|CP " SIZE_FORMAT_W(3), pin_count());
 438   st->cr();
 439 }
 440 
 441 // oop_iterate without closure and without cancellation.  always return true.
 442 bool ShenandoahHeapRegion::oop_fill_and_coalesce_wo_cancel() {
 443   HeapWord* obj_addr = resume_coalesce_and_fill();
 444 
 445   assert(!is_humongous(), "No need to fill or coalesce humongous regions");
 446   if (!is_active()) {
 447     end_preemptible_coalesce_and_fill();
 448     return true;
 449   }
 450 
 451   ShenandoahHeap* heap = ShenandoahHeap::heap();
 452   ShenandoahMarkingContext* marking_context = heap->marking_context();
 453   // All objects above TAMS are considered live even though their mark bits will not be set.  Note that young-
 454   // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
 455   // while the old-gen concurrent marking is ongoing.  These newly promoted objects will reside above TAMS
 456   // and will be treated as live during the current old-gen marking pass, even though they will not be
 457   // explicitly marked.
 458   HeapWord* t = marking_context->top_at_mark_start(this);
 459 
 460   // Expect marking to be completed before these threads invoke this service.
 461   assert(heap->active_generation()->is_mark_complete(), "sanity");
 462   while (obj_addr < t) {
 463     oop obj = cast_to_oop(obj_addr);
 464     if (marking_context->is_marked(obj)) {
 465       assert(obj->klass() != NULL, "klass should not be NULL");
 466       obj_addr += obj->size();
 467     } else {
 468       // Object is not marked.  Coalesce and fill dead object with dead neighbors.
 469       HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
 470       assert(next_marked_obj <= t, "next marked object cannot exceed top");
 471       size_t fill_size = next_marked_obj - obj_addr;
 472       ShenandoahHeap::fill_with_object(obj_addr, fill_size);
 473       heap->card_scan()->coalesce_objects(obj_addr, fill_size);
 474       obj_addr = next_marked_obj;
 475     }
 476   }
 477   // Mark that this region has been coalesced and filled
 478   end_preemptible_coalesce_and_fill();
 479   return true;
 480 }
 481 
 482 // oop_iterate without closure, return true if completed without cancellation
 483 bool ShenandoahHeapRegion::oop_fill_and_coalesce() {
 484   HeapWord* obj_addr = resume_coalesce_and_fill();
 485   // Consider yielding to cancel/preemption request after this many coalesce operations (skip marked, or coalesce free).
 486   const size_t preemption_stride = 128;
 487 
 488   assert(!is_humongous(), "No need to fill or coalesce humongous regions");
 489   if (!is_active()) {
 490     end_preemptible_coalesce_and_fill();
 491     return true;
 492   }
 493 
 494   ShenandoahHeap* heap = ShenandoahHeap::heap();
 495   ShenandoahMarkingContext* marking_context = heap->marking_context();
 496   // All objects above TAMS are considered live even though their mark bits will not be set.  Note that young-
 497   // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
 498   // while the old-gen concurrent marking is ongoing.  These newly promoted objects will reside above TAMS
 499   // and will be treated as live during the current old-gen marking pass, even though they will not be
 500   // explicitly marked.
 501   HeapWord* t = marking_context->top_at_mark_start(this);
 502 
 503   // Expect marking to be completed before these threads invoke this service.
 504   assert(heap->active_generation()->is_mark_complete(), "sanity");
 505 
 506   size_t ops_before_preempt_check = preemption_stride;
 507   while (obj_addr < t) {
 508     oop obj = cast_to_oop(obj_addr);
 509     if (marking_context->is_marked(obj)) {
 510       assert(obj->klass() != NULL, "klass should not be NULL");
 511       obj_addr += obj->size();
 512     } else {
 513       // Object is not marked.  Coalesce and fill dead object with dead neighbors.
 514       HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
 515       assert(next_marked_obj <= t, "next marked object cannot exceed top");
 516       size_t fill_size = next_marked_obj - obj_addr;
 517       ShenandoahHeap::fill_with_object(obj_addr, fill_size);
 518       heap->card_scan()->coalesce_objects(obj_addr, fill_size);
 519       obj_addr = next_marked_obj;
 520     }
 521     if (ops_before_preempt_check-- == 0) {
 522       if (heap->cancelled_gc()) {
 523         suspend_coalesce_and_fill(obj_addr);
 524         return false;
 525       }
 526       ops_before_preempt_check = preemption_stride;
 527     }
 528   }
 529   // Mark that this region has been coalesced and filled
 530   end_preemptible_coalesce_and_fill();
 531   return true;
 532 }
 533 
 534 void ShenandoahHeapRegion::global_oop_iterate_and_fill_dead(OopIterateClosure* blk) {
 535   if (!is_active()) return;
 536   if (is_humongous()) {
 537     // No need to fill dead within humongous regions.  Either the entire region is dead, or the entire region is
 538     // unchanged.  A humongous region holds no more than one humongous object.
 539     oop_iterate_humongous(blk);
 540   } else {
 541     global_oop_iterate_objects_and_fill_dead(blk);
 542   }
 543 }
 544 
 545 void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateClosure* blk) {
 546   assert(!is_humongous(), "no humongous region here");
 547   HeapWord* obj_addr = bottom();
 548 
 549   ShenandoahHeap* heap = ShenandoahHeap::heap();
 550   ShenandoahMarkingContext* marking_context = heap->marking_context();
 551   RememberedScanner* rem_set_scanner = heap->card_scan();
 552   // Objects allocated above TAMS are not marked, but are considered live for purposes of current GC efforts.
 553   HeapWord* t = marking_context->top_at_mark_start(this);
 554 
 555   assert(heap->active_generation()->is_mark_complete(), "sanity");
 556 
 557   while (obj_addr < t) {
 558     oop obj = cast_to_oop(obj_addr);
 559     if (marking_context->is_marked(obj)) {
 560       assert(obj->klass() != NULL, "klass should not be NULL");
 561       // when promoting an entire region, we have to register the marked objects as well
 562       obj_addr += obj->oop_iterate_size(blk);
 563     } else {
 564       // Object is not marked.  Coalesce and fill dead object with dead neighbors.
 565       HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
 566       assert(next_marked_obj <= t, "next marked object cannot exceed top");
 567       size_t fill_size = next_marked_obj - obj_addr;
 568       ShenandoahHeap::fill_with_object(obj_addr, fill_size);
 569 
 570       // coalesce_objects() unregisters all but first object subsumed within coalesced range.
 571       rem_set_scanner->coalesce_objects(obj_addr, fill_size);
 572       obj_addr = next_marked_obj;
 573     }
 574   }
 575 
 576   // Any object above TAMS and below top() is considered live.
 577   t = top();
 578   while (obj_addr < t) {
 579     oop obj = cast_to_oop(obj_addr);
 580     obj_addr += obj->oop_iterate_size(blk);
 581   }
 582 }
 583 
 584 // DO NOT CANCEL.  If this worker thread has accepted responsibility for scanning a particular range of addresses, it
 585 // must finish the work before it can be cancelled.
 586 void ShenandoahHeapRegion::oop_iterate_humongous_slice(OopIterateClosure* blk, bool dirty_only,
 587                                                        HeapWord* start, size_t words, bool write_table, bool is_concurrent) {
 588   assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards");
 589   assert(is_humongous(), "only humongous region here");
 590   ShenandoahHeap* heap = ShenandoahHeap::heap();
 591 
 592   // Find head.
 593   ShenandoahHeapRegion* r = humongous_start_region();
 594   assert(r->is_humongous_start(), "need humongous head here");
 595   assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words,
 596          "slice must be integral number of cards");
 597 
 598   oop obj = cast_to_oop(r->bottom());
 599   RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan();
 600   size_t card_index = scanner->card_index_for_addr(start);
 601   size_t num_cards = words / CardTable::card_size_in_words();
 602 
 603   if (dirty_only) {
 604     if (write_table) {
 605       while (num_cards-- > 0) {
 606         if (scanner->is_write_card_dirty(card_index++)) {
 607           obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
 608         }
 609         start += CardTable::card_size_in_words();
 610       }
 611     } else {
 612       while (num_cards-- > 0) {
 613         if (scanner->is_card_dirty(card_index++)) {
 614           obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
 615         }
 616         start += CardTable::card_size_in_words();
 617       }
 618     }
 619   } else {
 620     // Scan all data, regardless of whether cards are dirty
 621     obj->oop_iterate(blk, MemRegion(start, start + num_cards * CardTable::card_size_in_words()));
 622   }
 623 }
 624 
 625 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk, HeapWord* start, size_t words) {
 626   assert(is_humongous(), "only humongous region here");
 627   // Find head.
 628   ShenandoahHeapRegion* r = humongous_start_region();
 629   assert(r->is_humongous_start(), "need humongous head here");
 630   oop obj = cast_to_oop(r->bottom());
 631   obj->oop_iterate(blk, MemRegion(start, start + words));
 632 }
 633 
 634 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
 635   assert(is_humongous(), "only humongous region here");
 636   // Find head.
 637   ShenandoahHeapRegion* r = humongous_start_region();
 638   assert(r->is_humongous_start(), "need humongous head here");
 639   oop obj = cast_to_oop(r->bottom());
 640   obj->oop_iterate(blk, MemRegion(bottom(), top()));
 641 }
 642 
 643 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
 644   ShenandoahHeap* heap = ShenandoahHeap::heap();
 645   assert(is_humongous(), "Must be a part of the humongous region");
 646   size_t i = index();
 647   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
 648   while (!r->is_humongous_start()) {
 649     assert(i > 0, "Sanity");
 650     i--;
 651     r = heap->get_region(i);
 652     assert(r->is_humongous(), "Must be a part of the humongous region");
 653   }
 654   assert(r->is_humongous_start(), "Must be");
 655   return r;
 656 }
 657 
 658 void ShenandoahHeapRegion::recycle() {
 659   ShenandoahHeap* heap = ShenandoahHeap::heap();
 660 
 661   if (affiliation() == YOUNG_GENERATION) {
 662     heap->young_generation()->decrease_used(used());
 663   } else if (affiliation() == OLD_GENERATION) {
 664     heap->old_generation()->decrease_used(used());
 665   }
 666 
 667   set_top(bottom());
 668   clear_live_data();
 669 
 670   reset_alloc_metadata();
 671 
 672   heap->marking_context()->reset_top_at_mark_start(this);
 673   set_update_watermark(bottom());
 674 
 675   make_empty();
 676   set_affiliation(FREE);
 677 
 678   if (ZapUnusedHeapArea) {
 679     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
 680   }
 681 }
 682 
 683 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
 684   assert(MemRegion(bottom(), end()).contains(p),
 685          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
 686          p2i(p), p2i(bottom()), p2i(end()));
 687   if (p >= top()) {
 688     return top();
 689   } else {
 690     HeapWord* last = bottom();
 691     HeapWord* cur = last;
 692     while (cur <= p) {
 693       last = cur;
 694       cur += cast_to_oop(cur)->size();
 695     }
 696     shenandoah_assert_correct(NULL, cast_to_oop(last));

 907     evt.set_used(used());
 908     evt.set_from(_state);
 909     evt.set_to(to);
 910     evt.commit();
 911   }
 912   _state = to;
 913 }
 914 
 915 void ShenandoahHeapRegion::record_pin() {
 916   Atomic::add(&_critical_pins, (size_t)1);
 917 }
 918 
 919 void ShenandoahHeapRegion::record_unpin() {
 920   assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
 921   Atomic::sub(&_critical_pins, (size_t)1);
 922 }
 923 
 924 size_t ShenandoahHeapRegion::pin_count() const {
 925   return Atomic::load(&_critical_pins);
 926 }
 927 
 928 void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) {
 929   ShenandoahHeap* heap = ShenandoahHeap::heap();
 930 
 931   {
 932     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
 933     log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
 934                   ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
 935                   index(), affiliation_name(_affiliation), affiliation_name(new_affiliation),
 936                   p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this)));
 937   }
 938 
 939 #ifdef ASSERT
 940   {
 941     // During full gc, heap->complete_marking_context() is not valid, may equal nullptr.
 942     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
 943     size_t idx = this->index();
 944     HeapWord* top_bitmap = ctx->top_bitmap(this);
 945 
 946     assert(ctx->is_bitmap_clear_range(top_bitmap, _end),
 947            "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx,
 948            p2i(top_bitmap), p2i(_end));
 949   }
 950 #endif
 951 
 952   if (_affiliation == new_affiliation) {
 953     return;
 954   }
 955 
 956   if (!heap->mode()->is_generational()) {
 957     _affiliation = new_affiliation;
 958     return;
 959   }
 960 
 961   log_trace(gc)("Changing affiliation of region %zu from %s to %s",
 962     index(), affiliation_name(_affiliation), affiliation_name(new_affiliation));
 963 
 964   if (_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) {
 965     heap->young_generation()->decrement_affiliated_region_count();
 966   } else if (_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) {
 967     heap->old_generation()->decrement_affiliated_region_count();
 968   }
 969 
 970   switch (new_affiliation) {
 971     case FREE:
 972       assert(!has_live(), "Free region should not have live data");
 973       break;
 974     case YOUNG_GENERATION:
 975       reset_age();
 976       heap->young_generation()->increment_affiliated_region_count();
 977       break;
 978     case OLD_GENERATION:
 979       heap->old_generation()->increment_affiliated_region_count();
 980       break;
 981     default:
 982       ShouldNotReachHere();
 983       return;
 984   }
 985   _affiliation = new_affiliation;
 986 }
 987 
 988 size_t ShenandoahHeapRegion::promote_humongous() {
 989   ShenandoahHeap* heap = ShenandoahHeap::heap();
 990   ShenandoahMarkingContext* marking_context = heap->marking_context();
 991   assert(heap->active_generation()->is_mark_complete(), "sanity");
 992   assert(is_young(), "Only young regions can be promoted");
 993   assert(is_humongous_start(), "Should not promote humongous continuation in isolation");
 994   assert(age() >= InitialTenuringThreshold, "Only promote regions that are sufficiently aged");
 995 
 996   ShenandoahGeneration* old_generation = heap->old_generation();
 997   ShenandoahGeneration* young_generation = heap->young_generation();
 998 
 999   oop obj = cast_to_oop(bottom());
1000   assert(marking_context->is_marked(obj), "promoted humongous object should be alive");
1001 
1002   size_t spanned_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize);
1003   size_t index_limit = index() + spanned_regions;
1004 
1005   log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions);
1006 
1007   // Since this region may have served previously as OLD, it may hold obsolete object range info.
1008   heap->card_scan()->reset_object_range(bottom(), bottom() + spanned_regions * ShenandoahHeapRegion::region_size_words());
1009   // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation.
1010   heap->card_scan()->register_object_wo_lock(bottom());
1011 
1012   // For this region and each humongous continuation region spanned by this humongous object, change
1013   // affiliation to OLD_GENERATION and adjust the generation-use tallies.  The remnant of memory
1014   // in the last humongous region that is not spanned by obj is currently not used.
1015   for (size_t i = index(); i < index_limit; i++) {
1016     ShenandoahHeapRegion* r = heap->get_region(i);
1017     log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT,
1018                   r->index(), p2i(r->bottom()), p2i(r->top()));
1019     // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here
1020     r->set_affiliation(OLD_GENERATION);
1021     old_generation->increase_used(r->used());
1022     young_generation->decrease_used(r->used());
1023   }
1024   if (obj->is_typeArray()) {
1025     // Primitive arrays don't need to be scanned.  See above TODO question about requiring
1026     // region promotion at safepoint.
1027     log_debug(gc)("Clean cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
1028                   index(), p2i(bottom()), p2i(bottom() + obj->size()));
1029     heap->card_scan()->mark_range_as_clean(bottom(), obj->size());
1030   } else {
1031     log_debug(gc)("Dirty cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
1032                   index(), p2i(bottom()), p2i(bottom() + obj->size()));
1033     heap->card_scan()->mark_range_as_dirty(bottom(), obj->size());
1034   }
1035   return index_limit - index();
1036 }
< prev index next >