1 /*
2 * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shared/cardTable.hpp"
28 #include "gc/shared/space.hpp"
29 #include "gc/shared/tlab_globals.hpp"
30 #include "gc/shenandoah/shenandoahCardTable.hpp"
31 #include "gc/shenandoah/shenandoahFreeSet.hpp"
32 #include "gc/shenandoah/shenandoahGeneration.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
36 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
37 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
38 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
40 #include "jfr/jfrEvents.hpp"
41 #include "memory/allocation.hpp"
42 #include "memory/iterator.inline.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "oops/oop.inline.hpp"
46 #include "runtime/globals_extension.hpp"
47 #include "runtime/java.hpp"
48 #include "runtime/mutexLocker.hpp"
49 #include "runtime/os.hpp"
50 #include "runtime/safepoint.hpp"
51 #include "utilities/powerOfTwo.hpp"
52
53 size_t ShenandoahHeapRegion::RegionCount = 0;
54 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
55 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
56 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
57 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
58 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
59 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
60 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
61 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
62
63 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
64 _index(index),
65 _bottom(start),
66 _end(start + RegionSizeWords),
67 _new_top(nullptr),
68 _empty_time(os::elapsedTime()),
69 _top_before_promoted(nullptr),
70 _top_at_evac_start(start),
71 _state(committed ? _empty_committed : _empty_uncommitted),
72 _top(start),
73 _tlab_allocs(0),
74 _gclab_allocs(0),
75 _plab_allocs(0),
76 _live_data(0),
77 _critical_pins(0),
78 _mixed_candidate_garbage_words(0),
79 _update_watermark(start),
80 _age(0),
81 #ifdef SHENANDOAH_CENSUS_NOISE
82 _youth(0),
83 #endif // SHENANDOAH_CENSUS_NOISE
84 _needs_bitmap_reset(false)
85 {
86
87 assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
88 "invalid space boundaries");
89 if (ZapUnusedHeapArea && committed) {
90 SpaceMangler::mangle_region(MemRegion(_bottom, _end));
91 }
92 _recycling.unset();
93 _has_self_forwards.unset();
94 }
95
96 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
97 stringStream ss;
98 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(state()), method);
99 print_on(&ss);
100 fatal("%s", ss.freeze());
101 }
102
103 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) {
104 shenandoah_assert_heaplocked();
105 reset_age();
106 switch (state()) {
107 case _empty_uncommitted:
108 do_commit();
109 case _empty_committed:
110 assert(this->affiliation() == affiliation, "Region affiliation should already be established");
111 set_state(_regular);
112 case _regular:
113 case _pinned:
114 return;
115 default:
116 report_illegal_transition("regular allocation");
117 }
118 }
119
120 // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements
121 // behavior previously performed as a side effect of make_regular_bypass(). This is used by Full GC in non-generational
122 // modes to transition regions from FREE. Note that all non-free regions in single-generational modes are young.
123 void ShenandoahHeapRegion::make_affiliated_maybe() {
124 shenandoah_assert_heaplocked();
125 assert(!ShenandoahHeap::heap()->mode()->is_generational(), "Only call if non-generational");
126 switch (state()) {
127 case _empty_uncommitted:
128 case _empty_committed:
129 case _cset:
130 case _humongous_start:
131 case _humongous_cont:
132 if (affiliation() != YOUNG_GENERATION) {
133 set_affiliation(YOUNG_GENERATION);
134 }
135 return;
136 case _pinned_cset:
137 case _regular:
138 case _pinned:
139 return;
140 default:
141 assert(false, "Unexpected _state in make_affiliated_maybe");
142 }
143 }
144
145 void ShenandoahHeapRegion::make_regular_bypass() {
146 shenandoah_assert_heaplocked();
147 assert (ShenandoahHeap::heap()->is_full_gc_in_progress() ||
148 ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
149 "Only for STW GC");
150 reset_age();
151 switch (state()) {
152 case _empty_uncommitted:
153 do_commit();
154 case _empty_committed:
155 case _cset:
156 case _humongous_start:
157 case _humongous_cont:
158 set_state(_regular);
159 return;
160 case _pinned_cset:
161 set_state(_pinned);
162 return;
163 case _regular:
164 case _pinned:
165 return;
166 default:
167 report_illegal_transition("regular bypass");
168 }
169 }
170
171 void ShenandoahHeapRegion::make_humongous_start() {
172 shenandoah_assert_heaplocked();
173 reset_age();
174 switch (state()) {
175 case _empty_uncommitted:
176 do_commit();
177 case _empty_committed:
178 set_state(_humongous_start);
179 return;
180 default:
181 report_illegal_transition("humongous start allocation");
182 }
183 }
184
185 void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) {
186 shenandoah_assert_heaplocked();
187 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
188 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
189 set_affiliation(affiliation);
190 reset_age();
191 switch (state()) {
192 case _empty_committed:
193 case _regular:
194 case _humongous_start:
195 case _humongous_cont:
196 set_state(_humongous_start);
197 return;
198 default:
199 report_illegal_transition("humongous start bypass");
200 }
201 }
202
203 void ShenandoahHeapRegion::make_humongous_cont() {
204 shenandoah_assert_heaplocked();
205 reset_age();
206 switch (state()) {
207 case _empty_uncommitted:
208 do_commit();
209 case _empty_committed:
210 set_state(_humongous_cont);
211 return;
212 default:
213 report_illegal_transition("humongous continuation allocation");
214 }
215 }
216
217 void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) {
218 shenandoah_assert_heaplocked();
219 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
220 set_affiliation(affiliation);
221 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
222 reset_age();
223 switch (state()) {
224 case _empty_committed:
225 case _regular:
226 case _humongous_start:
227 case _humongous_cont:
228 set_state(_humongous_cont);
229 return;
230 default:
231 report_illegal_transition("humongous continuation bypass");
232 }
233 }
234
235 void ShenandoahHeapRegion::make_pinned() {
236 shenandoah_assert_heaplocked();
237 assert(pin_count() > 0, "Should have pins: %zu", pin_count());
238
239 switch (state()) {
240 case _regular:
241 set_state(_pinned);
242 case _pinned_cset:
243 case _pinned:
244 return;
245 case _humongous_start:
246 set_state(_pinned_humongous_start);
247 case _pinned_humongous_start:
248 return;
249 case _cset:
250 set_state(_pinned_cset);
251 return;
252 default:
253 report_illegal_transition("pinning");
254 }
255 }
256
257 void ShenandoahHeapRegion::make_unpinned() {
258 shenandoah_assert_heaplocked();
259 assert(pin_count() == 0, "Should not have pins: %zu", pin_count());
260
261 switch (state()) {
262 case _pinned:
263 assert(is_affiliated(), "Pinned region should be affiliated");
264 set_state(_regular);
265 return;
266 case _regular:
267 case _humongous_start:
268 return;
269 case _pinned_cset:
270 set_state(_cset);
271 return;
272 case _pinned_humongous_start:
273 set_state(_humongous_start);
274 return;
275 default:
276 report_illegal_transition("unpinning");
277 }
278 }
279
280 void ShenandoahHeapRegion::make_cset() {
281 shenandoah_assert_heaplocked();
282 // Leave age untouched. We need to consult the age when we are deciding whether to promote evacuated objects.
283 switch (state()) {
284 case _regular:
285 set_state(_cset);
286 case _cset:
287 return;
288 default:
289 report_illegal_transition("cset");
290 }
291 }
292
293 void ShenandoahHeapRegion::make_trash() {
294 shenandoah_assert_heaplocked();
295 reset_age();
296 switch (state()) {
297 case _humongous_start:
298 case _humongous_cont:
299 {
300 // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim
301 // its used memory. At recycle time, we no longer recognize this as a humongous region.
302 decrement_humongous_waste();
303 }
304 case _cset:
305 // Reclaiming cset regions
306 case _regular:
307 // Immediate region reclaim
308 set_state(_trash);
309 return;
310 default:
311 report_illegal_transition("trashing");
312 }
313 }
314
315 void ShenandoahHeapRegion::make_trash_immediate() {
316 make_trash();
317
318 // On this path, we know there are no marked objects in the region,
319 // tell marking context about it to bypass bitmap resets.
320 const ShenandoahHeap* heap = ShenandoahHeap::heap();
321 assert(heap->generation_for(affiliation())->is_mark_complete(), "Marking should be complete here.");
322 heap->marking_context()->reset_top_bitmap(this);
323 }
324
325 void ShenandoahHeapRegion::make_empty() {
326 reset_age();
327 CENSUS_NOISE(clear_youth();)
328 switch (state()) {
329 case _trash:
330 set_state(_empty_committed);
331 _empty_time = os::elapsedTime();
332 return;
333 default:
334 report_illegal_transition("emptying");
335 }
336 }
337
338 void ShenandoahHeapRegion::make_uncommitted() {
339 shenandoah_assert_heaplocked();
340 switch (state()) {
341 case _empty_committed:
342 do_uncommit();
343 set_state(_empty_uncommitted);
344 return;
345 default:
346 report_illegal_transition("uncommiting");
347 }
348 }
349
350 void ShenandoahHeapRegion::make_committed_bypass() {
351 shenandoah_assert_heaplocked();
352 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
353
354 switch (state()) {
355 case _empty_uncommitted:
356 do_commit();
357 set_state(_empty_committed);
358 return;
359 default:
360 report_illegal_transition("commit bypass");
361 }
362 }
363
364 void ShenandoahHeapRegion::reset_alloc_metadata() {
365 _tlab_allocs = 0;
366 _gclab_allocs = 0;
367 _plab_allocs = 0;
368 }
369
370 size_t ShenandoahHeapRegion::get_shared_allocs() const {
371 return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize;
372 }
373
374 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
375 return _tlab_allocs * HeapWordSize;
376 }
377
378 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
379 return _gclab_allocs * HeapWordSize;
380 }
381
382 size_t ShenandoahHeapRegion::get_plab_allocs() const {
383 return _plab_allocs * HeapWordSize;
384 }
385
386 void ShenandoahHeapRegion::set_live_data(size_t s) {
387 assert(Thread::current()->is_VM_thread(), "by VM thread");
388 _live_data.store_relaxed(s >> LogHeapWordSize);
389 }
390
391 void ShenandoahHeapRegion::print_on(outputStream* st) const {
392 st->print("|");
393 st->print("%5zu", this->_index);
394
395 switch (state()) {
396 case _empty_uncommitted:
397 st->print("|EU ");
398 break;
399 case _empty_committed:
400 st->print("|EC ");
401 break;
402 case _regular:
403 st->print("|R ");
404 break;
405 case _humongous_start:
406 st->print("|H ");
407 break;
408 case _pinned_humongous_start:
409 st->print("|HP ");
410 break;
411 case _humongous_cont:
412 st->print("|HC ");
413 break;
414 case _cset:
415 st->print("|CS ");
416 break;
417 case _trash:
418 st->print("|TR ");
419 break;
420 case _pinned:
421 st->print("|P ");
422 break;
423 case _pinned_cset:
424 st->print("|CSP");
425 break;
426 default:
427 ShouldNotReachHere();
428 }
429
430 st->print("|%s", shenandoah_affiliation_code(affiliation()));
431
432 #define SHR_PTR_FORMAT "%12" PRIxPTR
433
434 st->print("|BTE " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT,
435 p2i(bottom()), p2i(top()), p2i(end()));
436 st->print("|TAMS " SHR_PTR_FORMAT,
437 p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
438 st->print("|UWM " SHR_PTR_FORMAT,
439 p2i(_update_watermark.load_relaxed()));
440 st->print("|U %5zu%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
441 st->print("|T %5zu%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
442 st->print("|G %5zu%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
443 if (ShenandoahHeap::heap()->mode()->is_generational()) {
444 st->print("|P %5zu%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs()));
445 }
446 st->print("|S %5zu%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
447 st->print("|L %5zu%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
448 st->print("|CP %3zu", pin_count());
449 st->cr();
450
451 #undef SHR_PTR_FORMAT
452 }
453
454 // oop_iterate without closure, return true if completed without cancellation
455 bool ShenandoahHeapRegion::oop_coalesce_and_fill(bool cancellable) {
456
457 assert(!is_humongous(), "No need to fill or coalesce humongous regions");
458 if (!is_active()) {
459 end_preemptible_coalesce_and_fill();
460 return true;
461 }
462
463 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
464 ShenandoahMarkingContext* marking_context = heap->marking_context();
465
466 // Expect marking to be completed for the old generation before we fill in unmarked objects
467 assert(heap->old_generation()->is_mark_complete(), "sanity");
468 assert(is_old(), "Only need to coalesce and fill old regions");
469
470 // All objects above TAMS are considered live even though their mark bits will not be set. Note that young-
471 // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
472 // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS
473 // and will be treated as live during the current old-gen marking pass, even though they will not be
474 // explicitly marked.
475 HeapWord* t = marking_context->top_at_mark_start(this);
476
477 // Resume coalesce and fill from this address
478 HeapWord* obj_addr = resume_coalesce_and_fill();
479
480 while (obj_addr < t) {
481 oop obj = cast_to_oop(obj_addr);
482 if (marking_context->is_marked(obj)) {
483 assert(obj->klass() != nullptr, "klass should not be nullptr");
484 obj_addr += obj->size();
485 } else {
486 // Object is not marked. Coalesce and fill dead object with dead neighbors.
487 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
488 assert(next_marked_obj <= t, "next marked object cannot exceed top");
489 size_t fill_size = next_marked_obj - obj_addr;
490 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
491 ShenandoahHeap::fill_with_object(obj_addr, fill_size);
492 heap->old_generation()->card_scan()->coalesce_objects(obj_addr, fill_size);
493 obj_addr = next_marked_obj;
494 }
495 if (cancellable && heap->cancelled_gc()) {
496 suspend_coalesce_and_fill(obj_addr);
497 return false;
498 }
499 }
500 // Mark that this region has been coalesced and filled
501 end_preemptible_coalesce_and_fill();
502 return true;
503 }
504
505 size_t get_card_count(size_t words) {
506 assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards");
507 assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words,
508 "slice must be integral number of cards");
509 return words / CardTable::card_size_in_words();
510 }
511
512 void ShenandoahHeapRegion::oop_iterate_humongous_slice_dirty(OopIterateClosure* blk,
513 HeapWord* start, size_t words, bool write_table) const {
514 assert(is_humongous(), "only humongous region here");
515
516 ShenandoahHeapRegion* r = humongous_start_region();
517 oop obj = cast_to_oop(r->bottom());
518 size_t num_cards = get_card_count(words);
519
520 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
521 ShenandoahScanRemembered* scanner = heap->old_generation()->card_scan();
522 size_t card_index = scanner->card_index_for_addr(start);
523 if (write_table) {
524 while (num_cards-- > 0) {
525 if (scanner->is_write_card_dirty(card_index++)) {
526 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
527 }
528 start += CardTable::card_size_in_words();
529 }
530 } else {
531 while (num_cards-- > 0) {
532 if (scanner->is_card_dirty(card_index++)) {
533 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
534 }
535 start += CardTable::card_size_in_words();
536 }
537 }
538 }
539
540 void ShenandoahHeapRegion::oop_iterate_humongous_slice_all(OopIterateClosure* cl, HeapWord* start, size_t words) const {
541 assert(is_humongous(), "only humongous region here");
542
543 ShenandoahHeapRegion* r = humongous_start_region();
544 oop obj = cast_to_oop(r->bottom());
545
546 // Scan all data, regardless of whether cards are dirty
547 obj->oop_iterate(cl, MemRegion(start, start + words));
548 }
549
550 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
551 ShenandoahHeap* heap = ShenandoahHeap::heap();
552 assert(is_humongous(), "Must be a part of the humongous region");
553 size_t i = index();
554 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
555 while (!r->is_humongous_start()) {
556 assert(i > 0, "Sanity");
557 i--;
558 r = heap->get_region(i);
559 assert(r->is_humongous(), "Must be a part of the humongous region");
560 }
561 assert(r->is_humongous_start(), "Must be");
562 return r;
563 }
564
565
566 void ShenandoahHeapRegion::recycle_internal() {
567 assert(_recycling.is_set() && is_trash(), "Wrong state");
568 ShenandoahHeap* heap = ShenandoahHeap::heap();
569
570 _top_at_evac_start = _bottom;
571 _mixed_candidate_garbage_words = 0;
572 clear_live_data();
573 reset_alloc_metadata();
574 heap->marking_context()->reset_top_at_mark_start(this);
575 set_update_watermark(bottom());
576 clear_has_self_forwards();
577 if (is_old()) {
578 heap->old_generation()->clear_cards_for(this);
579 }
580
581 if (ZapUnusedHeapArea) {
582 SpaceMangler::mangle_region(MemRegion(bottom(), top()));
583 }
584 set_top(bottom());
585 set_affiliation(FREE);
586
587 // Lastly, set region state to empty
588 make_empty();
589 }
590
591 // Upon return, this region has been recycled. We try to recycle it.
592 // We may fail if some other thread recycled it before we do.
593 void ShenandoahHeapRegion::try_recycle_under_lock() {
594 shenandoah_assert_heaplocked();
595 if (!is_trash()) {
596 return;
597 }
598 if (_recycling.try_set()) {
599 if (is_trash()) {
600 // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as
601 // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism
602 // for triggering heuristics. It greatly simplifies and reduces the locking overhead required
603 // by more time-precise accounting of these details.
604 recycle_internal();
605 }
606 _recycling.unset();
607 } else {
608 // Ensure recycling is unset before returning to mutator to continue memory allocation.
609 // Otherwise, the mutator might see region as fully recycled and might change its affiliation only to have
610 // the racing GC worker thread overwrite its affiliation to FREE.
611 while (_recycling.is_set()) {
612 if (os::is_MP()) {
613 SpinPause();
614 } else {
615 os::naked_yield();
616 }
617 }
618 assert(!is_trash(), "Must not");
619 }
620 }
621
622 // Note that return from try_recycle() does not mean the region has been recycled. It only means that
623 // some GC worker thread has taken responsibility to recycle the region, eventually.
624 void ShenandoahHeapRegion::try_recycle() {
625 shenandoah_assert_not_heaplocked();
626 if (!is_trash()) {
627 return;
628 }
629 if (_recycling.try_set()) {
630 // Double check region state after win the race to set recycling flag
631 if (is_trash()) {
632 // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as
633 // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism
634 // for triggering and pacing heuristics. It greatly simplifies and reduces the locking overhead required
635 // by more time-precise accounting of these details.
636 recycle_internal();
637 }
638 _recycling.unset();
639 }
640 }
641
642 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
643 assert(MemRegion(bottom(), end()).contains(p),
644 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
645 p2i(p), p2i(bottom()), p2i(end()));
646 if (p >= top()) {
647 return top();
648 } else {
649 HeapWord* last = bottom();
650 HeapWord* cur = last;
651 while (cur <= p) {
652 last = cur;
653 cur += cast_to_oop(cur)->size();
654 }
655 shenandoah_assert_correct(nullptr, cast_to_oop(last));
656 return last;
657 }
658 }
659
660 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
661 assert(MemRegion(bottom(), end()).contains(p),
662 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
663 p2i(p), p2i(bottom()), p2i(end()));
664 if (p < top()) {
665 return cast_to_oop(p)->size();
666 } else {
667 assert(p == top(), "just checking");
668 return pointer_delta(end(), (HeapWord*) p);
669 }
670 }
671
672 size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
673 // Generational Shenandoah needs this alignment for card tables.
674 if (strcmp(ShenandoahGCMode, "generational") == 0) {
675 max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint());
676 }
677
678 size_t region_size;
679 if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
680 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
681 // for usual heap sizes. Do not depend on initial_heap_size here.
682 region_size = max_heap_size / ShenandoahTargetNumRegions;
683
684 // Now make sure that we don't go over or under our limits.
685 region_size = MAX2(MIN_REGION_SIZE, region_size);
686 region_size = MIN2(MAX_REGION_SIZE, region_size);
687 } else {
688 if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
689 err_msg message("Max heap size (%zu%s) is too low to afford the minimum number "
690 "of regions (%zu) of requested size (%zu%s).",
691 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
692 MIN_NUM_REGIONS,
693 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
694 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
695 }
696 if (ShenandoahRegionSize < MIN_REGION_SIZE) {
697 err_msg message("Heap region size (%zu%s) should be larger than min region size (%zu%s).",
698 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
699 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
700 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
701 }
702 if (ShenandoahRegionSize > MAX_REGION_SIZE) {
703 err_msg message("Heap region size (%zu%s) should be lower than max region size (%zu%s).",
704 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
705 byte_size_in_proper_unit(MAX_REGION_SIZE), proper_unit_for_byte_size(MAX_REGION_SIZE));
706 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
707 }
708 region_size = ShenandoahRegionSize;
709 }
710
711 // Make sure region size and heap size are page aligned.
712 // If large pages are used, we ensure that region size is aligned to large page size if
713 // heap size is large enough to accommodate minimal number of regions. Otherwise, we align
714 // region size to regular page size.
715
716 // Figure out page size to use, and aligns up heap to page size
717 size_t page_size = os::vm_page_size();
718 if (UseLargePages) {
719 size_t large_page_size = os::large_page_size();
720 max_heap_size = align_up(max_heap_size, large_page_size);
721 if ((max_heap_size / align_up(region_size, large_page_size)) >= MIN_NUM_REGIONS) {
722 page_size = large_page_size;
723 } else {
724 // Should have been checked during argument initialization
725 assert(!ShenandoahUncommit, "Uncommit requires region size aligns to large page size");
726 }
727 } else {
728 max_heap_size = align_up(max_heap_size, page_size);
729 }
730
731 // Align region size to page size
732 region_size = align_up(region_size, page_size);
733
734 int region_size_log = log2i(region_size);
735 // Recalculate the region size to make sure it's a power of
736 // 2. This means that region_size is the largest power of 2 that's
737 // <= what we've calculated so far.
738 region_size = size_t(1) << region_size_log;
739
740 // Now, set up the globals.
741 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
742 RegionSizeBytesShift = (size_t)region_size_log;
743
744 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
745 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
746
747 guarantee(RegionSizeBytes == 0, "we should only set it once");
748 RegionSizeBytes = region_size;
749 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
750 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
751
752 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
753 RegionSizeWordsMask = RegionSizeWords - 1;
754
755 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
756 RegionSizeBytesMask = RegionSizeBytes - 1;
757
758 guarantee(RegionCount == 0, "we should only set it once");
759 RegionCount = align_up(max_heap_size, RegionSizeBytes) / RegionSizeBytes;
760 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
761
762 // Limit TLAB size for better startup behavior and more equitable distribution of memory between contending mutator threads.
763 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
764 MaxTLABSizeWords = align_down(MIN2(RegionSizeWords, MAX2(RegionSizeWords / 32, (size_t) (256 * 1024) / HeapWordSize)),
765 MinObjAlignment);
766
767 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
768 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
769 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
770
771 return max_heap_size;
772 }
773
774 void ShenandoahHeapRegion::do_commit() {
775 ShenandoahHeap* heap = ShenandoahHeap::heap();
776 if (!heap->is_heap_region_special()) {
777 os::commit_memory_or_exit((char*) bottom(), RegionSizeBytes, false, "Unable to commit region");
778 }
779 if (!heap->is_bitmap_region_special()) {
780 heap->commit_bitmap_slice(this);
781 }
782 if (AlwaysPreTouch) {
783 os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size());
784 }
785 if (ZapUnusedHeapArea) {
786 SpaceMangler::mangle_region(MemRegion(bottom(), end()));
787 }
788 heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
789 }
790
791 void ShenandoahHeapRegion::do_uncommit() {
792 ShenandoahHeap* heap = ShenandoahHeap::heap();
793 if (!heap->is_heap_region_special()) {
794 os::uncommit_memory((char *) bottom(), RegionSizeBytes);
795 }
796 if (!heap->is_bitmap_region_special()) {
797 heap->uncommit_bitmap_slice(this);
798 }
799 heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
800 }
801
802 void ShenandoahHeapRegion::set_state(RegionState to) {
803 EventShenandoahHeapRegionStateChange evt;
804 if (evt.should_commit()){
805 evt.set_index((unsigned) index());
806 evt.set_start((uintptr_t)bottom());
807 evt.set_used(used());
808 evt.set_from(state());
809 evt.set_to(to);
810 evt.commit();
811 }
812 _state.release_store(to);
813 }
814
815 void ShenandoahHeapRegion::record_pin() {
816 _critical_pins.add_then_fetch((size_t)1);
817 }
818
819 void ShenandoahHeapRegion::record_unpin() {
820 assert(pin_count() > 0, "Region %zu should have non-zero pins", index());
821 _critical_pins.sub_then_fetch((size_t)1);
822 }
823
824 size_t ShenandoahHeapRegion::pin_count() const {
825 return _critical_pins.load_relaxed();
826 }
827
828 void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) {
829 ShenandoahHeap* heap = ShenandoahHeap::heap();
830
831 ShenandoahAffiliation region_affiliation = heap->region_affiliation(this);
832 ShenandoahMarkingContext* const ctx = heap->marking_context();
833 {
834 log_debug(gc)("Setting affiliation of Region %zu from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
835 ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
836 index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation),
837 p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark.load_relaxed()), p2i(ctx->top_bitmap(this)));
838 }
839
840 #ifdef ASSERT
841 {
842 size_t idx = this->index();
843 HeapWord* top_bitmap = ctx->top_bitmap(this);
844
845 assert(ctx->is_bitmap_range_within_region_clear(top_bitmap, _end),
846 "Region %zu, bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx,
847 p2i(top_bitmap), p2i(_end));
848 }
849 #endif
850
851 if (region_affiliation == new_affiliation) {
852 return;
853 }
854
855 if (!heap->mode()->is_generational()) {
856 log_trace(gc)("Changing affiliation of region %zu from %s to %s",
857 index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation));
858 heap->set_affiliation(this, new_affiliation);
859 return;
860 }
861
862 switch (new_affiliation) {
863 case FREE:
864 assert(!has_live(), "Free region should not have live data");
865 break;
866 case YOUNG_GENERATION:
867 reset_age();
868 break;
869 case OLD_GENERATION:
870 break;
871 default:
872 ShouldNotReachHere();
873 return;
874 }
875 heap->set_affiliation(this, new_affiliation);
876 }
877
878 void ShenandoahHeapRegion::decrement_humongous_waste() {
879 assert(is_humongous(), "Should only use this for humongous regions");
880 size_t waste_bytes = free();
881 if (waste_bytes > 0) {
882 ShenandoahHeap* heap = ShenandoahHeap::heap();
883 heap->free_set()->decrease_humongous_waste_for_regular_bypass(this, waste_bytes);
884 }
885 }