1 /* 2 * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "gc/shared/cardTable.hpp" 29 #include "gc/shared/space.hpp" 30 #include "gc/shared/tlab_globals.hpp" 31 #include "gc/shenandoah/shenandoahCardTable.hpp" 32 #include "gc/shenandoah/shenandoahFreeSet.hpp" 33 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" 34 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 35 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 36 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 37 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 38 #include "gc/shenandoah/shenandoahGeneration.hpp" 39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 40 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 41 #include "jfr/jfrEvents.hpp" 42 #include "memory/allocation.hpp" 43 #include "memory/iterator.inline.hpp" 44 #include "memory/resourceArea.hpp" 45 #include "memory/universe.hpp" 46 #include "oops/oop.inline.hpp" 47 #include "runtime/atomic.hpp" 48 #include "runtime/globals_extension.hpp" 49 #include "runtime/java.hpp" 50 #include "runtime/mutexLocker.hpp" 51 #include "runtime/os.hpp" 52 #include "runtime/safepoint.hpp" 53 #include "utilities/powerOfTwo.hpp" 54 55 size_t ShenandoahHeapRegion::RegionCount = 0; 56 size_t ShenandoahHeapRegion::RegionSizeBytes = 0; 57 size_t ShenandoahHeapRegion::RegionSizeWords = 0; 58 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0; 59 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0; 60 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0; 61 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0; 62 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0; 63 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0; 64 65 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) : 66 _index(index), 67 _bottom(start), 68 _end(start + RegionSizeWords), 69 _new_top(nullptr), 70 _empty_time(os::elapsedTime()), 71 _top_before_promoted(nullptr), 72 _state(committed ? _empty_committed : _empty_uncommitted), 73 _top(start), 74 _tlab_allocs(0), 75 _gclab_allocs(0), 76 _plab_allocs(0), 77 _live_data(0), 78 _critical_pins(0), 79 _update_watermark(start), 80 _age(0) 81 #ifdef SHENANDOAH_CENSUS_NOISE 82 , _youth(0) 83 #endif // SHENANDOAH_CENSUS_NOISE 84 { 85 86 assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), 87 "invalid space boundaries"); 88 if (ZapUnusedHeapArea && committed) { 89 SpaceMangler::mangle_region(MemRegion(_bottom, _end)); 90 } 91 } 92 93 void ShenandoahHeapRegion::report_illegal_transition(const char *method) { 94 stringStream ss; 95 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method); 96 print_on(&ss); 97 fatal("%s", ss.freeze()); 98 } 99 100 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) { 101 shenandoah_assert_heaplocked(); 102 reset_age(); 103 switch (_state) { 104 case _empty_uncommitted: 105 do_commit(); 106 case _empty_committed: 107 assert(this->affiliation() == affiliation, "Region affiliation should already be established"); 108 set_state(_regular); 109 case _regular: 110 case _pinned: 111 return; 112 default: 113 report_illegal_transition("regular allocation"); 114 } 115 } 116 117 // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements 118 // behavior previously performed as a side effect of make_regular_bypass(). This is used by Full GC in non-generational 119 // modes to transition regions from FREE. Note that all non-free regions in single-generational modes are young. 120 void ShenandoahHeapRegion::make_affiliated_maybe() { 121 shenandoah_assert_heaplocked(); 122 assert(!ShenandoahHeap::heap()->mode()->is_generational(), "Only call if non-generational"); 123 switch (_state) { 124 case _empty_uncommitted: 125 case _empty_committed: 126 case _cset: 127 case _humongous_start: 128 case _humongous_cont: 129 if (affiliation() != YOUNG_GENERATION) { 130 set_affiliation(YOUNG_GENERATION); 131 } 132 return; 133 case _pinned_cset: 134 case _regular: 135 case _pinned: 136 return; 137 default: 138 assert(false, "Unexpected _state in make_affiliated_maybe"); 139 } 140 } 141 142 void ShenandoahHeapRegion::make_regular_bypass() { 143 shenandoah_assert_heaplocked(); 144 assert (!Universe::is_fully_initialized() || 145 ShenandoahHeap::heap()->is_full_gc_in_progress() || 146 ShenandoahHeap::heap()->is_degenerated_gc_in_progress(), 147 "Only for STW GC or when Universe is initializing (CDS)"); 148 reset_age(); 149 switch (_state) { 150 case _empty_uncommitted: 151 do_commit(); 152 case _empty_committed: 153 case _cset: 154 case _humongous_start: 155 case _humongous_cont: 156 if (_state == _humongous_start || _state == _humongous_cont) { 157 // CDS allocates chunks of the heap to fill with regular objects. The allocator 158 // will dutifully track any waste in the unused portion of the last region. Once 159 // CDS has finished initializing the objects, it will convert these regions to 160 // regular regions. The 'waste' in the last region is no longer wasted at this point, 161 // so we must stop treating it as such. 162 decrement_humongous_waste(); 163 } 164 set_state(_regular); 165 return; 166 case _pinned_cset: 167 set_state(_pinned); 168 return; 169 case _regular: 170 case _pinned: 171 return; 172 default: 173 report_illegal_transition("regular bypass"); 174 } 175 } 176 177 void ShenandoahHeapRegion::make_humongous_start() { 178 shenandoah_assert_heaplocked(); 179 reset_age(); 180 switch (_state) { 181 case _empty_uncommitted: 182 do_commit(); 183 case _empty_committed: 184 set_state(_humongous_start); 185 return; 186 default: 187 report_illegal_transition("humongous start allocation"); 188 } 189 } 190 191 void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) { 192 shenandoah_assert_heaplocked(); 193 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); 194 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end. 195 set_affiliation(affiliation); 196 reset_age(); 197 switch (_state) { 198 case _empty_committed: 199 case _regular: 200 case _humongous_start: 201 case _humongous_cont: 202 set_state(_humongous_start); 203 return; 204 default: 205 report_illegal_transition("humongous start bypass"); 206 } 207 } 208 209 void ShenandoahHeapRegion::make_humongous_cont() { 210 shenandoah_assert_heaplocked(); 211 reset_age(); 212 switch (_state) { 213 case _empty_uncommitted: 214 do_commit(); 215 case _empty_committed: 216 set_state(_humongous_cont); 217 return; 218 default: 219 report_illegal_transition("humongous continuation allocation"); 220 } 221 } 222 223 void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) { 224 shenandoah_assert_heaplocked(); 225 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); 226 set_affiliation(affiliation); 227 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end. 228 reset_age(); 229 switch (_state) { 230 case _empty_committed: 231 case _regular: 232 case _humongous_start: 233 case _humongous_cont: 234 set_state(_humongous_cont); 235 return; 236 default: 237 report_illegal_transition("humongous continuation bypass"); 238 } 239 } 240 241 void ShenandoahHeapRegion::make_pinned() { 242 shenandoah_assert_heaplocked(); 243 assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count()); 244 245 switch (_state) { 246 case _regular: 247 set_state(_pinned); 248 case _pinned_cset: 249 case _pinned: 250 return; 251 case _humongous_start: 252 set_state(_pinned_humongous_start); 253 case _pinned_humongous_start: 254 return; 255 case _cset: 256 _state = _pinned_cset; 257 return; 258 default: 259 report_illegal_transition("pinning"); 260 } 261 } 262 263 void ShenandoahHeapRegion::make_unpinned() { 264 shenandoah_assert_heaplocked(); 265 assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count()); 266 267 switch (_state) { 268 case _pinned: 269 assert(is_affiliated(), "Pinned region should be affiliated"); 270 set_state(_regular); 271 return; 272 case _regular: 273 case _humongous_start: 274 return; 275 case _pinned_cset: 276 set_state(_cset); 277 return; 278 case _pinned_humongous_start: 279 set_state(_humongous_start); 280 return; 281 default: 282 report_illegal_transition("unpinning"); 283 } 284 } 285 286 void ShenandoahHeapRegion::make_cset() { 287 shenandoah_assert_heaplocked(); 288 // Leave age untouched. We need to consult the age when we are deciding whether to promote evacuated objects. 289 switch (_state) { 290 case _regular: 291 set_state(_cset); 292 case _cset: 293 return; 294 default: 295 report_illegal_transition("cset"); 296 } 297 } 298 299 void ShenandoahHeapRegion::make_trash() { 300 shenandoah_assert_heaplocked(); 301 reset_age(); 302 switch (_state) { 303 case _humongous_start: 304 case _humongous_cont: 305 { 306 // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim 307 // its used memory. At recycle time, we no longer recognize this as a humongous region. 308 decrement_humongous_waste(); 309 } 310 case _cset: 311 // Reclaiming cset regions 312 case _regular: 313 // Immediate region reclaim 314 set_state(_trash); 315 return; 316 default: 317 report_illegal_transition("trashing"); 318 } 319 } 320 321 void ShenandoahHeapRegion::make_trash_immediate() { 322 make_trash(); 323 324 // On this path, we know there are no marked objects in the region, 325 // tell marking context about it to bypass bitmap resets. 326 assert(ShenandoahHeap::heap()->gc_generation()->is_mark_complete(), "Marking should be complete here."); 327 shenandoah_assert_generations_reconciled(); 328 ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); 329 } 330 331 void ShenandoahHeapRegion::make_empty() { 332 shenandoah_assert_heaplocked(); 333 reset_age(); 334 CENSUS_NOISE(clear_youth();) 335 switch (_state) { 336 case _trash: 337 set_state(_empty_committed); 338 _empty_time = os::elapsedTime(); 339 return; 340 default: 341 report_illegal_transition("emptying"); 342 } 343 } 344 345 void ShenandoahHeapRegion::make_uncommitted() { 346 shenandoah_assert_heaplocked(); 347 switch (_state) { 348 case _empty_committed: 349 do_uncommit(); 350 set_state(_empty_uncommitted); 351 return; 352 default: 353 report_illegal_transition("uncommiting"); 354 } 355 } 356 357 void ShenandoahHeapRegion::make_committed_bypass() { 358 shenandoah_assert_heaplocked(); 359 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); 360 361 switch (_state) { 362 case _empty_uncommitted: 363 do_commit(); 364 set_state(_empty_committed); 365 return; 366 default: 367 report_illegal_transition("commit bypass"); 368 } 369 } 370 371 void ShenandoahHeapRegion::reset_alloc_metadata() { 372 _tlab_allocs = 0; 373 _gclab_allocs = 0; 374 _plab_allocs = 0; 375 } 376 377 size_t ShenandoahHeapRegion::get_shared_allocs() const { 378 return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize; 379 } 380 381 size_t ShenandoahHeapRegion::get_tlab_allocs() const { 382 return _tlab_allocs * HeapWordSize; 383 } 384 385 size_t ShenandoahHeapRegion::get_gclab_allocs() const { 386 return _gclab_allocs * HeapWordSize; 387 } 388 389 size_t ShenandoahHeapRegion::get_plab_allocs() const { 390 return _plab_allocs * HeapWordSize; 391 } 392 393 void ShenandoahHeapRegion::set_live_data(size_t s) { 394 assert(Thread::current()->is_VM_thread(), "by VM thread"); 395 _live_data = (s >> LogHeapWordSize); 396 } 397 398 void ShenandoahHeapRegion::print_on(outputStream* st) const { 399 st->print("|"); 400 st->print(SIZE_FORMAT_W(5), this->_index); 401 402 switch (_state) { 403 case _empty_uncommitted: 404 st->print("|EU "); 405 break; 406 case _empty_committed: 407 st->print("|EC "); 408 break; 409 case _regular: 410 st->print("|R "); 411 break; 412 case _humongous_start: 413 st->print("|H "); 414 break; 415 case _pinned_humongous_start: 416 st->print("|HP "); 417 break; 418 case _humongous_cont: 419 st->print("|HC "); 420 break; 421 case _cset: 422 st->print("|CS "); 423 break; 424 case _trash: 425 st->print("|TR "); 426 break; 427 case _pinned: 428 st->print("|P "); 429 break; 430 case _pinned_cset: 431 st->print("|CSP"); 432 break; 433 default: 434 ShouldNotReachHere(); 435 } 436 437 st->print("|%s", shenandoah_affiliation_code(affiliation())); 438 439 #define SHR_PTR_FORMAT "%12" PRIxPTR 440 441 st->print("|BTE " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT, 442 p2i(bottom()), p2i(top()), p2i(end())); 443 st->print("|TAMS " SHR_PTR_FORMAT, 444 p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this)))); 445 st->print("|UWM " SHR_PTR_FORMAT, 446 p2i(_update_watermark)); 447 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used())); 448 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs())); 449 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs())); 450 if (ShenandoahHeap::heap()->mode()->is_generational()) { 451 st->print("|P " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs())); 452 } 453 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs())); 454 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes())); 455 st->print("|CP " SIZE_FORMAT_W(3), pin_count()); 456 st->cr(); 457 458 #undef SHR_PTR_FORMAT 459 } 460 461 // oop_iterate without closure, return true if completed without cancellation 462 bool ShenandoahHeapRegion::oop_coalesce_and_fill(bool cancellable) { 463 464 assert(!is_humongous(), "No need to fill or coalesce humongous regions"); 465 if (!is_active()) { 466 end_preemptible_coalesce_and_fill(); 467 return true; 468 } 469 470 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 471 ShenandoahMarkingContext* marking_context = heap->marking_context(); 472 473 // Expect marking to be completed before these threads invoke this service. 474 assert(heap->gc_generation()->is_mark_complete(), "sanity"); 475 shenandoah_assert_generations_reconciled(); 476 477 // All objects above TAMS are considered live even though their mark bits will not be set. Note that young- 478 // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen 479 // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS 480 // and will be treated as live during the current old-gen marking pass, even though they will not be 481 // explicitly marked. 482 HeapWord* t = marking_context->top_at_mark_start(this); 483 484 // Resume coalesce and fill from this address 485 HeapWord* obj_addr = resume_coalesce_and_fill(); 486 487 while (obj_addr < t) { 488 oop obj = cast_to_oop(obj_addr); 489 if (marking_context->is_marked(obj)) { 490 assert(obj->klass() != nullptr, "klass should not be nullptr"); 491 obj_addr += obj->size(); 492 } else { 493 // Object is not marked. Coalesce and fill dead object with dead neighbors. 494 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); 495 assert(next_marked_obj <= t, "next marked object cannot exceed top"); 496 size_t fill_size = next_marked_obj - obj_addr; 497 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size"); 498 ShenandoahHeap::fill_with_object(obj_addr, fill_size); 499 heap->old_generation()->card_scan()->coalesce_objects(obj_addr, fill_size); 500 obj_addr = next_marked_obj; 501 } 502 if (cancellable && heap->cancelled_gc()) { 503 suspend_coalesce_and_fill(obj_addr); 504 return false; 505 } 506 } 507 // Mark that this region has been coalesced and filled 508 end_preemptible_coalesce_and_fill(); 509 return true; 510 } 511 512 size_t get_card_count(size_t words) { 513 assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards"); 514 assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words, 515 "slice must be integral number of cards"); 516 return words / CardTable::card_size_in_words(); 517 } 518 519 void ShenandoahHeapRegion::oop_iterate_humongous_slice_dirty(OopIterateClosure* blk, 520 HeapWord* start, size_t words, bool write_table) const { 521 assert(is_humongous(), "only humongous region here"); 522 523 ShenandoahHeapRegion* r = humongous_start_region(); 524 oop obj = cast_to_oop(r->bottom()); 525 size_t num_cards = get_card_count(words); 526 527 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 528 ShenandoahScanRemembered* scanner = heap->old_generation()->card_scan(); 529 size_t card_index = scanner->card_index_for_addr(start); 530 if (write_table) { 531 while (num_cards-- > 0) { 532 if (scanner->is_write_card_dirty(card_index++)) { 533 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words())); 534 } 535 start += CardTable::card_size_in_words(); 536 } 537 } else { 538 while (num_cards-- > 0) { 539 if (scanner->is_card_dirty(card_index++)) { 540 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words())); 541 } 542 start += CardTable::card_size_in_words(); 543 } 544 } 545 } 546 547 void ShenandoahHeapRegion::oop_iterate_humongous_slice_all(OopIterateClosure* cl, HeapWord* start, size_t words) const { 548 assert(is_humongous(), "only humongous region here"); 549 550 ShenandoahHeapRegion* r = humongous_start_region(); 551 oop obj = cast_to_oop(r->bottom()); 552 553 // Scan all data, regardless of whether cards are dirty 554 obj->oop_iterate(cl, MemRegion(start, start + words)); 555 } 556 557 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { 558 ShenandoahHeap* heap = ShenandoahHeap::heap(); 559 assert(is_humongous(), "Must be a part of the humongous region"); 560 size_t i = index(); 561 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this); 562 while (!r->is_humongous_start()) { 563 assert(i > 0, "Sanity"); 564 i--; 565 r = heap->get_region(i); 566 assert(r->is_humongous(), "Must be a part of the humongous region"); 567 } 568 assert(r->is_humongous_start(), "Must be"); 569 return r; 570 } 571 572 void ShenandoahHeapRegion::recycle() { 573 shenandoah_assert_heaplocked(); 574 ShenandoahHeap* heap = ShenandoahHeap::heap(); 575 ShenandoahGeneration* generation = heap->generation_for(affiliation()); 576 577 heap->decrease_used(generation, used()); 578 generation->decrement_affiliated_region_count(); 579 580 set_top(bottom()); 581 clear_live_data(); 582 reset_alloc_metadata(); 583 584 heap->marking_context()->reset_top_at_mark_start(this); 585 586 set_update_watermark(bottom()); 587 588 make_empty(); 589 590 set_affiliation(FREE); 591 if (ZapUnusedHeapArea) { 592 SpaceMangler::mangle_region(MemRegion(bottom(), end())); 593 } 594 } 595 596 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const { 597 assert(MemRegion(bottom(), end()).contains(p), 598 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", 599 p2i(p), p2i(bottom()), p2i(end())); 600 if (p >= top()) { 601 return top(); 602 } else { 603 HeapWord* last = bottom(); 604 HeapWord* cur = last; 605 while (cur <= p) { 606 last = cur; 607 cur += cast_to_oop(cur)->size(); 608 } 609 shenandoah_assert_correct(nullptr, cast_to_oop(last)); 610 return last; 611 } 612 } 613 614 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const { 615 assert(MemRegion(bottom(), end()).contains(p), 616 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", 617 p2i(p), p2i(bottom()), p2i(end())); 618 if (p < top()) { 619 return cast_to_oop(p)->size(); 620 } else { 621 assert(p == top(), "just checking"); 622 return pointer_delta(end(), (HeapWord*) p); 623 } 624 } 625 626 size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) { 627 // Absolute minimums we should not ever break. 628 static const size_t MIN_REGION_SIZE = 256*K; 629 630 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) { 631 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE); 632 } 633 634 // Generational Shenandoah needs this alignment for card tables. 635 if (strcmp(ShenandoahGCMode, "generational") == 0) { 636 max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint()); 637 } 638 639 size_t region_size; 640 if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) { 641 if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) { 642 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number " 643 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).", 644 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size), 645 MIN_NUM_REGIONS, 646 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize)); 647 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message); 648 } 649 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) { 650 err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).", 651 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize), 652 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE)); 653 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message); 654 } 655 if (ShenandoahMinRegionSize < MinTLABSize) { 656 err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).", 657 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize), 658 byte_size_in_proper_unit(MinTLABSize), proper_unit_for_byte_size(MinTLABSize)); 659 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message); 660 } 661 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) { 662 err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).", 663 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize), 664 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE)); 665 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message); 666 } 667 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) { 668 err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).", 669 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize), 670 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize)); 671 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message); 672 } 673 674 // We rapidly expand to max_heap_size in most scenarios, so that is the measure 675 // for usual heap sizes. Do not depend on initial_heap_size here. 676 region_size = max_heap_size / ShenandoahTargetNumRegions; 677 678 // Now make sure that we don't go over or under our limits. 679 region_size = MAX2(ShenandoahMinRegionSize, region_size); 680 region_size = MIN2(ShenandoahMaxRegionSize, region_size); 681 682 } else { 683 if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) { 684 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number " 685 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).", 686 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size), 687 MIN_NUM_REGIONS, 688 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize)); 689 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message); 690 } 691 if (ShenandoahRegionSize < ShenandoahMinRegionSize) { 692 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).", 693 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize), 694 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize)); 695 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message); 696 } 697 if (ShenandoahRegionSize > ShenandoahMaxRegionSize) { 698 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).", 699 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize), 700 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize)); 701 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message); 702 } 703 region_size = ShenandoahRegionSize; 704 } 705 706 // Make sure region size and heap size are page aligned. 707 // If large pages are used, we ensure that region size is aligned to large page size if 708 // heap size is large enough to accommodate minimal number of regions. Otherwise, we align 709 // region size to regular page size. 710 711 // Figure out page size to use, and aligns up heap to page size 712 size_t page_size = os::vm_page_size(); 713 if (UseLargePages) { 714 size_t large_page_size = os::large_page_size(); 715 max_heap_size = align_up(max_heap_size, large_page_size); 716 if ((max_heap_size / align_up(region_size, large_page_size)) >= MIN_NUM_REGIONS) { 717 page_size = large_page_size; 718 } else { 719 // Should have been checked during argument initialization 720 assert(!ShenandoahUncommit, "Uncommit requires region size aligns to large page size"); 721 } 722 } else { 723 max_heap_size = align_up(max_heap_size, page_size); 724 } 725 726 // Align region size to page size 727 region_size = align_up(region_size, page_size); 728 729 int region_size_log = log2i(region_size); 730 // Recalculate the region size to make sure it's a power of 731 // 2. This means that region_size is the largest power of 2 that's 732 // <= what we've calculated so far. 733 region_size = size_t(1) << region_size_log; 734 735 // Now, set up the globals. 736 guarantee(RegionSizeBytesShift == 0, "we should only set it once"); 737 RegionSizeBytesShift = (size_t)region_size_log; 738 739 guarantee(RegionSizeWordsShift == 0, "we should only set it once"); 740 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize; 741 742 guarantee(RegionSizeBytes == 0, "we should only set it once"); 743 RegionSizeBytes = region_size; 744 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize; 745 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity"); 746 747 guarantee(RegionSizeWordsMask == 0, "we should only set it once"); 748 RegionSizeWordsMask = RegionSizeWords - 1; 749 750 guarantee(RegionSizeBytesMask == 0, "we should only set it once"); 751 RegionSizeBytesMask = RegionSizeBytes - 1; 752 753 guarantee(RegionCount == 0, "we should only set it once"); 754 RegionCount = align_up(max_heap_size, RegionSizeBytes) / RegionSizeBytes; 755 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions"); 756 757 guarantee(MaxTLABSizeWords == 0, "we should only set it once"); 758 MaxTLABSizeWords = align_down(RegionSizeWords, MinObjAlignment); 759 760 guarantee(MaxTLABSizeBytes == 0, "we should only set it once"); 761 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize; 762 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger"); 763 764 return max_heap_size; 765 } 766 767 void ShenandoahHeapRegion::do_commit() { 768 ShenandoahHeap* heap = ShenandoahHeap::heap(); 769 if (!heap->is_heap_region_special() && !os::commit_memory((char *) bottom(), RegionSizeBytes, false)) { 770 report_java_out_of_memory("Unable to commit region"); 771 } 772 if (!heap->commit_bitmap_slice(this)) { 773 report_java_out_of_memory("Unable to commit bitmaps for region"); 774 } 775 if (AlwaysPreTouch) { 776 os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size()); 777 } 778 heap->increase_committed(ShenandoahHeapRegion::region_size_bytes()); 779 } 780 781 void ShenandoahHeapRegion::do_uncommit() { 782 ShenandoahHeap* heap = ShenandoahHeap::heap(); 783 if (!heap->is_heap_region_special() && !os::uncommit_memory((char *) bottom(), RegionSizeBytes)) { 784 report_java_out_of_memory("Unable to uncommit region"); 785 } 786 if (!heap->uncommit_bitmap_slice(this)) { 787 report_java_out_of_memory("Unable to uncommit bitmaps for region"); 788 } 789 heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes()); 790 } 791 792 void ShenandoahHeapRegion::set_state(RegionState to) { 793 EventShenandoahHeapRegionStateChange evt; 794 if (evt.should_commit()){ 795 evt.set_index((unsigned) index()); 796 evt.set_start((uintptr_t)bottom()); 797 evt.set_used(used()); 798 evt.set_from(_state); 799 evt.set_to(to); 800 evt.commit(); 801 } 802 _state = to; 803 } 804 805 void ShenandoahHeapRegion::record_pin() { 806 Atomic::add(&_critical_pins, (size_t)1); 807 } 808 809 void ShenandoahHeapRegion::record_unpin() { 810 assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index()); 811 Atomic::sub(&_critical_pins, (size_t)1); 812 } 813 814 size_t ShenandoahHeapRegion::pin_count() const { 815 return Atomic::load(&_critical_pins); 816 } 817 818 void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) { 819 ShenandoahHeap* heap = ShenandoahHeap::heap(); 820 821 ShenandoahAffiliation region_affiliation = heap->region_affiliation(this); 822 { 823 ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); 824 log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT 825 ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT, 826 index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation), 827 p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this))); 828 } 829 830 #ifdef ASSERT 831 { 832 // During full gc, heap->complete_marking_context() is not valid, may equal nullptr. 833 ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); 834 size_t idx = this->index(); 835 HeapWord* top_bitmap = ctx->top_bitmap(this); 836 837 assert(ctx->is_bitmap_range_within_region_clear(top_bitmap, _end), 838 "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx, 839 p2i(top_bitmap), p2i(_end)); 840 } 841 #endif 842 843 if (region_affiliation == new_affiliation) { 844 return; 845 } 846 847 if (!heap->mode()->is_generational()) { 848 log_trace(gc)("Changing affiliation of region %zu from %s to %s", 849 index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation)); 850 heap->set_affiliation(this, new_affiliation); 851 return; 852 } 853 854 switch (new_affiliation) { 855 case FREE: 856 assert(!has_live(), "Free region should not have live data"); 857 break; 858 case YOUNG_GENERATION: 859 reset_age(); 860 break; 861 case OLD_GENERATION: 862 break; 863 default: 864 ShouldNotReachHere(); 865 return; 866 } 867 heap->set_affiliation(this, new_affiliation); 868 } 869 870 void ShenandoahHeapRegion::decrement_humongous_waste() const { 871 assert(is_humongous(), "Should only use this for humongous regions"); 872 size_t waste_bytes = free(); 873 if (waste_bytes > 0) { 874 ShenandoahHeap* heap = ShenandoahHeap::heap(); 875 ShenandoahGeneration* generation = heap->generation_for(affiliation()); 876 heap->decrease_humongous_waste(generation, waste_bytes); 877 } 878 }