1 /*
2 * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "gc/shared/cardTable.hpp"
29 #include "gc/shared/space.inline.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "gc/shenandoah/shenandoahCardTable.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
36 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
37 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
38 #include "gc/shenandoah/shenandoahGeneration.hpp"
39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
40 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
41 #include "jfr/jfrEvents.hpp"
42 #include "memory/allocation.hpp"
43 #include "memory/iterator.inline.hpp"
44 #include "memory/resourceArea.hpp"
45 #include "memory/universe.hpp"
46 #include "oops/oop.inline.hpp"
47 #include "runtime/atomic.hpp"
48 #include "runtime/globals_extension.hpp"
49 #include "runtime/java.hpp"
50 #include "runtime/mutexLocker.hpp"
51 #include "runtime/os.hpp"
52 #include "runtime/safepoint.hpp"
53 #include "utilities/powerOfTwo.hpp"
54
55
56 size_t ShenandoahHeapRegion::RegionCount = 0;
57 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
58 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
59 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
60 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
61 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
62 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
63 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
64 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
65 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
66 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
67
68 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
69 _index(index),
70 _bottom(start),
71 _end(start + RegionSizeWords),
72 _new_top(nullptr),
73 _empty_time(os::elapsedTime()),
74 _top_before_promoted(nullptr),
75 _state(committed ? _empty_committed : _empty_uncommitted),
76 _top(start),
77 _tlab_allocs(0),
78 _gclab_allocs(0),
79 _plab_allocs(0),
80 _live_data(0),
81 _critical_pins(0),
82 _update_watermark(start),
83 _age(0)
84 #ifdef SHENANDOAH_CENSUS_NOISE
85 , _youth(0)
86 #endif // SHENANDOAH_CENSUS_NOISE
87 {
88
89 assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
90 "invalid space boundaries");
91 if (ZapUnusedHeapArea && committed) {
92 SpaceMangler::mangle_region(MemRegion(_bottom, _end));
93 }
94 }
95
96 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
97 stringStream ss;
98 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
99 print_on(&ss);
100 fatal("%s", ss.freeze());
101 }
102
103 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) {
104 shenandoah_assert_heaplocked();
105 reset_age();
106 switch (_state) {
107 case _empty_uncommitted:
108 do_commit();
109 case _empty_committed:
110 assert(this->affiliation() == affiliation, "Region affiliation should already be established");
111 set_state(_regular);
112 case _regular:
113 case _pinned:
114 return;
115 default:
116 report_illegal_transition("regular allocation");
117 }
118 }
119
120 // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements
121 // behavior previously performed as a side effect of make_regular_bypass().
122 void ShenandoahHeapRegion::make_young_maybe() {
123 shenandoah_assert_heaplocked();
124 switch (_state) {
125 case _empty_uncommitted:
126 case _empty_committed:
127 case _cset:
128 case _humongous_start:
129 case _humongous_cont:
130 if (affiliation() != YOUNG_GENERATION) {
131 if (is_old()) {
132 ShenandoahHeap::heap()->old_generation()->decrement_affiliated_region_count();
133 }
134 set_affiliation(YOUNG_GENERATION);
135 ShenandoahHeap::heap()->young_generation()->increment_affiliated_region_count();
136 }
137 return;
138 case _pinned_cset:
139 case _regular:
140 case _pinned:
141 return;
142 default:
143 assert(false, "Unexpected _state in make_young_maybe");
144 }
145 }
146
147 void ShenandoahHeapRegion::make_regular_bypass() {
148 shenandoah_assert_heaplocked();
149 assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
150 "only for full or degen GC");
151 reset_age();
152 switch (_state) {
153 case _empty_uncommitted:
154 do_commit();
155 case _empty_committed:
156 case _cset:
157 case _humongous_start:
158 case _humongous_cont:
159 set_state(_regular);
160 return;
161 case _pinned_cset:
162 set_state(_pinned);
163 return;
164 case _regular:
165 case _pinned:
166 return;
167 default:
168 report_illegal_transition("regular bypass");
169 }
170 }
171
172 void ShenandoahHeapRegion::make_humongous_start() {
173 shenandoah_assert_heaplocked();
174 reset_age();
175 switch (_state) {
176 case _empty_uncommitted:
177 do_commit();
178 case _empty_committed:
179 set_state(_humongous_start);
180 return;
181 default:
182 report_illegal_transition("humongous start allocation");
183 }
184 }
185
186 void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) {
187 shenandoah_assert_heaplocked();
188 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
189 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
190 set_affiliation(affiliation);
191 reset_age();
192 switch (_state) {
193 case _empty_committed:
194 case _regular:
195 case _humongous_start:
196 case _humongous_cont:
197 set_state(_humongous_start);
198 return;
199 default:
200 report_illegal_transition("humongous start bypass");
201 }
202 }
203
204 void ShenandoahHeapRegion::make_humongous_cont() {
205 shenandoah_assert_heaplocked();
206 reset_age();
207 switch (_state) {
208 case _empty_uncommitted:
209 do_commit();
210 case _empty_committed:
211 set_state(_humongous_cont);
212 return;
213 default:
214 report_illegal_transition("humongous continuation allocation");
215 }
216 }
217
218 void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) {
219 shenandoah_assert_heaplocked();
220 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
221 set_affiliation(affiliation);
222 // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
223 reset_age();
224 switch (_state) {
225 case _empty_committed:
226 case _regular:
227 case _humongous_start:
228 case _humongous_cont:
229 set_state(_humongous_cont);
230 return;
231 default:
232 report_illegal_transition("humongous continuation bypass");
233 }
234 }
235
236 void ShenandoahHeapRegion::make_pinned() {
237 shenandoah_assert_heaplocked();
238 assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
239
240 switch (_state) {
241 case _regular:
242 set_state(_pinned);
243 case _pinned_cset:
244 case _pinned:
245 return;
246 case _humongous_start:
247 set_state(_pinned_humongous_start);
248 case _pinned_humongous_start:
249 return;
250 case _cset:
251 _state = _pinned_cset;
252 return;
253 default:
254 report_illegal_transition("pinning");
255 }
256 }
257
258 void ShenandoahHeapRegion::make_unpinned() {
259 shenandoah_assert_heaplocked();
260 assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
261
262 switch (_state) {
263 case _pinned:
264 assert(is_affiliated(), "Pinned region should be affiliated");
265 set_state(_regular);
266 return;
267 case _regular:
268 case _humongous_start:
269 return;
270 case _pinned_cset:
271 set_state(_cset);
272 return;
273 case _pinned_humongous_start:
274 set_state(_humongous_start);
275 return;
276 default:
277 report_illegal_transition("unpinning");
278 }
279 }
280
281 void ShenandoahHeapRegion::make_cset() {
282 shenandoah_assert_heaplocked();
283 // Leave age untouched. We need to consult the age when we are deciding whether to promote evacuated objects.
284 switch (_state) {
285 case _regular:
286 set_state(_cset);
287 case _cset:
288 return;
289 default:
290 report_illegal_transition("cset");
291 }
292 }
293
294 void ShenandoahHeapRegion::make_trash() {
295 shenandoah_assert_heaplocked();
296 reset_age();
297 switch (_state) {
298 case _humongous_start:
299 case _humongous_cont:
300 {
301 // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim
302 // its used memory. At recycle time, we no longer recognize this as a humongous region.
303 decrement_humongous_waste();
304 }
305 case _cset:
306 // Reclaiming cset regions
307 case _regular:
308 // Immediate region reclaim
309 set_state(_trash);
310 return;
311 default:
312 report_illegal_transition("trashing");
313 }
314 }
315
316 void ShenandoahHeapRegion::make_trash_immediate() {
317 make_trash();
318
319 // On this path, we know there are no marked objects in the region,
320 // tell marking context about it to bypass bitmap resets.
321 assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here.");
322 ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this);
323 }
324
325 void ShenandoahHeapRegion::make_empty() {
326 shenandoah_assert_heaplocked();
327 reset_age();
328 CENSUS_NOISE(clear_youth();)
329 switch (_state) {
330 case _trash:
331 set_state(_empty_committed);
332 _empty_time = os::elapsedTime();
333 return;
334 default:
335 report_illegal_transition("emptying");
336 }
337 }
338
339 void ShenandoahHeapRegion::make_uncommitted() {
340 shenandoah_assert_heaplocked();
341 switch (_state) {
342 case _empty_committed:
343 do_uncommit();
344 set_state(_empty_uncommitted);
345 return;
346 default:
347 report_illegal_transition("uncommiting");
348 }
349 }
350
351 void ShenandoahHeapRegion::make_committed_bypass() {
352 shenandoah_assert_heaplocked();
353 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
354
355 switch (_state) {
356 case _empty_uncommitted:
357 do_commit();
358 set_state(_empty_committed);
359 return;
360 default:
361 report_illegal_transition("commit bypass");
362 }
363 }
364
365 void ShenandoahHeapRegion::reset_alloc_metadata() {
366 _tlab_allocs = 0;
367 _gclab_allocs = 0;
368 _plab_allocs = 0;
369 }
370
371 size_t ShenandoahHeapRegion::get_shared_allocs() const {
372 return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize;
373 }
374
375 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
376 return _tlab_allocs * HeapWordSize;
377 }
378
379 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
380 return _gclab_allocs * HeapWordSize;
381 }
382
383 size_t ShenandoahHeapRegion::get_plab_allocs() const {
384 return _plab_allocs * HeapWordSize;
385 }
386
387 void ShenandoahHeapRegion::set_live_data(size_t s) {
388 assert(Thread::current()->is_VM_thread(), "by VM thread");
389 _live_data = (s >> LogHeapWordSize);
390 }
391
392 void ShenandoahHeapRegion::print_on(outputStream* st) const {
393 st->print("|");
394 st->print(SIZE_FORMAT_W(5), this->_index);
395
396 switch (_state) {
397 case _empty_uncommitted:
398 st->print("|EU ");
399 break;
400 case _empty_committed:
401 st->print("|EC ");
402 break;
403 case _regular:
404 st->print("|R ");
405 break;
406 case _humongous_start:
407 st->print("|H ");
408 break;
409 case _pinned_humongous_start:
410 st->print("|HP ");
411 break;
412 case _humongous_cont:
413 st->print("|HC ");
414 break;
415 case _cset:
416 st->print("|CS ");
417 break;
418 case _trash:
419 st->print("|TR ");
420 break;
421 case _pinned:
422 st->print("|P ");
423 break;
424 case _pinned_cset:
425 st->print("|CSP");
426 break;
427 default:
428 ShouldNotReachHere();
429 }
430
431 st->print("|%s", shenandoah_affiliation_code(affiliation()));
432
433 #define SHR_PTR_FORMAT "%12" PRIxPTR
434
435 st->print("|BTE " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT,
436 p2i(bottom()), p2i(top()), p2i(end()));
437 st->print("|TAMS " SHR_PTR_FORMAT,
438 p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
439 st->print("|UWM " SHR_PTR_FORMAT,
440 p2i(_update_watermark));
441 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
442 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
443 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
444 if (ShenandoahHeap::heap()->mode()->is_generational()) {
445 st->print("|P " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs()));
446 }
447 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
448 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
449 st->print("|CP " SIZE_FORMAT_W(3), pin_count());
450 st->cr();
451
452 #undef SHR_PTR_FORMAT
453 }
454
455 // oop_iterate without closure and without cancellation. always return true.
456 bool ShenandoahHeapRegion::oop_fill_and_coalesce_without_cancel() {
457 HeapWord* obj_addr = resume_coalesce_and_fill();
458
459 assert(!is_humongous(), "No need to fill or coalesce humongous regions");
460 if (!is_active()) {
461 end_preemptible_coalesce_and_fill();
462 return true;
463 }
464
465 ShenandoahHeap* heap = ShenandoahHeap::heap();
466 ShenandoahMarkingContext* marking_context = heap->marking_context();
467 // All objects above TAMS are considered live even though their mark bits will not be set. Note that young-
468 // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
469 // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS
470 // and will be treated as live during the current old-gen marking pass, even though they will not be
471 // explicitly marked.
472 HeapWord* t = marking_context->top_at_mark_start(this);
473
474 // Expect marking to be completed before these threads invoke this service.
475 assert(heap->active_generation()->is_mark_complete(), "sanity");
476 while (obj_addr < t) {
477 oop obj = cast_to_oop(obj_addr);
478 if (marking_context->is_marked(obj)) {
479 assert(obj->klass() != nullptr, "klass should not be nullptr");
480 obj_addr += obj->size();
481 } else {
482 // Object is not marked. Coalesce and fill dead object with dead neighbors.
483 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
484 assert(next_marked_obj <= t, "next marked object cannot exceed top");
485 size_t fill_size = next_marked_obj - obj_addr;
486 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size");
487 ShenandoahHeap::fill_with_object(obj_addr, fill_size);
488 heap->card_scan()->coalesce_objects(obj_addr, fill_size);
489 obj_addr = next_marked_obj;
490 }
491 }
492 // Mark that this region has been coalesced and filled
493 end_preemptible_coalesce_and_fill();
494 return true;
495 }
496
497 // oop_iterate without closure, return true if completed without cancellation
498 bool ShenandoahHeapRegion::oop_fill_and_coalesce() {
499 HeapWord* obj_addr = resume_coalesce_and_fill();
500 // Consider yielding to cancel/preemption request after this many coalesce operations (skip marked, or coalesce free).
501 const size_t preemption_stride = 128;
502
503 assert(!is_humongous(), "No need to fill or coalesce humongous regions");
504 if (!is_active()) {
505 end_preemptible_coalesce_and_fill();
506 return true;
507 }
508
509 ShenandoahHeap* heap = ShenandoahHeap::heap();
510 ShenandoahMarkingContext* marking_context = heap->marking_context();
511 // All objects above TAMS are considered live even though their mark bits will not be set. Note that young-
512 // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
513 // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS
514 // and will be treated as live during the current old-gen marking pass, even though they will not be
515 // explicitly marked.
516 HeapWord* t = marking_context->top_at_mark_start(this);
517
518 // Expect marking to be completed before these threads invoke this service.
519 assert(heap->active_generation()->is_mark_complete(), "sanity");
520
521 size_t ops_before_preempt_check = preemption_stride;
522 while (obj_addr < t) {
523 oop obj = cast_to_oop(obj_addr);
524 if (marking_context->is_marked(obj)) {
525 assert(obj->klass() != nullptr, "klass should not be nullptr");
526 obj_addr += obj->size();
527 } else {
528 // Object is not marked. Coalesce and fill dead object with dead neighbors.
529 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
530 assert(next_marked_obj <= t, "next marked object cannot exceed top");
531 size_t fill_size = next_marked_obj - obj_addr;
532 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
533 ShenandoahHeap::fill_with_object(obj_addr, fill_size);
534 heap->card_scan()->coalesce_objects(obj_addr, fill_size);
535 obj_addr = next_marked_obj;
536 }
537 if (ops_before_preempt_check-- == 0) {
538 if (heap->cancelled_gc()) {
539 suspend_coalesce_and_fill(obj_addr);
540 return false;
541 }
542 ops_before_preempt_check = preemption_stride;
543 }
544 }
545 // Mark that this region has been coalesced and filled
546 end_preemptible_coalesce_and_fill();
547 return true;
548 }
549
550 void ShenandoahHeapRegion::global_oop_iterate_and_fill_dead(OopIterateClosure* blk) {
551 if (!is_active()) return;
552 if (is_humongous()) {
553 // No need to fill dead within humongous regions. Either the entire region is dead, or the entire region is
554 // unchanged. A humongous region holds no more than one humongous object.
555 oop_iterate_humongous(blk);
556 } else {
557 global_oop_iterate_objects_and_fill_dead(blk);
558 }
559 }
560
561 void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateClosure* blk) {
562 assert(!is_humongous(), "no humongous region here");
563 HeapWord* obj_addr = bottom();
564
565 ShenandoahHeap* heap = ShenandoahHeap::heap();
566 ShenandoahMarkingContext* marking_context = heap->marking_context();
567 RememberedScanner* rem_set_scanner = heap->card_scan();
568 // Objects allocated above TAMS are not marked, but are considered live for purposes of current GC efforts.
569 HeapWord* t = marking_context->top_at_mark_start(this);
570
571 assert(heap->active_generation()->is_mark_complete(), "sanity");
572
573 while (obj_addr < t) {
574 oop obj = cast_to_oop(obj_addr);
575 if (marking_context->is_marked(obj)) {
576 assert(obj->klass() != nullptr, "klass should not be nullptr");
577 // when promoting an entire region, we have to register the marked objects as well
578 obj_addr += obj->oop_iterate_size(blk);
579 } else {
580 // Object is not marked. Coalesce and fill dead object with dead neighbors.
581 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
582 assert(next_marked_obj <= t, "next marked object cannot exceed top");
583 size_t fill_size = next_marked_obj - obj_addr;
584 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size");
585 ShenandoahHeap::fill_with_object(obj_addr, fill_size);
586 // coalesce_objects() unregisters all but first object subsumed within coalesced range.
587 rem_set_scanner->coalesce_objects(obj_addr, fill_size);
588 obj_addr = next_marked_obj;
589 }
590 }
591
592 // Any object above TAMS and below top() is considered live.
593 t = top();
594 while (obj_addr < t) {
595 oop obj = cast_to_oop(obj_addr);
596 obj_addr += obj->oop_iterate_size(blk);
597 }
598 }
599
600 // DO NOT CANCEL. If this worker thread has accepted responsibility for scanning a particular range of addresses, it
601 // must finish the work before it can be cancelled.
602 void ShenandoahHeapRegion::oop_iterate_humongous_slice(OopIterateClosure* blk, bool dirty_only,
603 HeapWord* start, size_t words, bool write_table) {
604 assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards");
605 assert(is_humongous(), "only humongous region here");
606 ShenandoahHeap* heap = ShenandoahHeap::heap();
607
608 // Find head.
609 ShenandoahHeapRegion* r = humongous_start_region();
610 assert(r->is_humongous_start(), "need humongous head here");
611 assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words,
612 "slice must be integral number of cards");
613
614 oop obj = cast_to_oop(r->bottom());
615 RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan();
616 size_t card_index = scanner->card_index_for_addr(start);
617 size_t num_cards = words / CardTable::card_size_in_words();
618
619 if (dirty_only) {
620 if (write_table) {
621 while (num_cards-- > 0) {
622 if (scanner->is_write_card_dirty(card_index++)) {
623 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
624 }
625 start += CardTable::card_size_in_words();
626 }
627 } else {
628 while (num_cards-- > 0) {
629 if (scanner->is_card_dirty(card_index++)) {
630 obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
631 }
632 start += CardTable::card_size_in_words();
633 }
634 }
635 } else {
636 // Scan all data, regardless of whether cards are dirty
637 obj->oop_iterate(blk, MemRegion(start, start + num_cards * CardTable::card_size_in_words()));
638 }
639 }
640
641 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk, HeapWord* start, size_t words) {
642 assert(is_humongous(), "only humongous region here");
643 // Find head.
644 ShenandoahHeapRegion* r = humongous_start_region();
645 assert(r->is_humongous_start(), "need humongous head here");
646 oop obj = cast_to_oop(r->bottom());
647 obj->oop_iterate(blk, MemRegion(start, start + words));
648 }
649
650 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
651 assert(is_humongous(), "only humongous region here");
652 // Find head.
653 ShenandoahHeapRegion* r = humongous_start_region();
654 assert(r->is_humongous_start(), "need humongous head here");
655 oop obj = cast_to_oop(r->bottom());
656 obj->oop_iterate(blk, MemRegion(bottom(), top()));
657 }
658
659 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
660 ShenandoahHeap* heap = ShenandoahHeap::heap();
661 assert(is_humongous(), "Must be a part of the humongous region");
662 size_t i = index();
663 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
664 while (!r->is_humongous_start()) {
665 assert(i > 0, "Sanity");
666 i--;
667 r = heap->get_region(i);
668 assert(r->is_humongous(), "Must be a part of the humongous region");
669 }
670 assert(r->is_humongous_start(), "Must be");
671 return r;
672 }
673
674 void ShenandoahHeapRegion::recycle() {
675 shenandoah_assert_heaplocked();
676 ShenandoahHeap* heap = ShenandoahHeap::heap();
677 ShenandoahGeneration* generation = heap->generation_for(affiliation());
678 heap->decrease_used(generation, used());
679
680 set_top(bottom());
681 clear_live_data();
682
683 reset_alloc_metadata();
684
685 heap->marking_context()->reset_top_at_mark_start(this);
686 set_update_watermark(bottom());
687
688 make_empty();
689 ShenandoahHeap::heap()->generation_for(affiliation())->decrement_affiliated_region_count();
690 set_affiliation(FREE);
691 if (ZapUnusedHeapArea) {
692 SpaceMangler::mangle_region(MemRegion(bottom(), end()));
693 }
694 }
695
696 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
697 assert(MemRegion(bottom(), end()).contains(p),
698 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
699 p2i(p), p2i(bottom()), p2i(end()));
700 if (p >= top()) {
701 return top();
702 } else {
703 HeapWord* last = bottom();
704 HeapWord* cur = last;
705 while (cur <= p) {
706 last = cur;
707 cur += cast_to_oop(cur)->size();
708 }
709 shenandoah_assert_correct(nullptr, cast_to_oop(last));
710 return last;
711 }
712 }
713
714 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
715 assert(MemRegion(bottom(), end()).contains(p),
716 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
717 p2i(p), p2i(bottom()), p2i(end()));
718 if (p < top()) {
719 return cast_to_oop(p)->size();
720 } else {
721 assert(p == top(), "just checking");
722 return pointer_delta(end(), (HeapWord*) p);
723 }
724 }
725
726 size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
727 // Absolute minimums we should not ever break.
728 static const size_t MIN_REGION_SIZE = 256*K;
729
730 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
731 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
732 }
733
734 // Generational Shenandoah needs this alignment for card tables.
735 if (strcmp(ShenandoahGCMode, "generational") == 0) {
736 max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint());
737 }
738
739 size_t region_size;
740 if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
741 if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
742 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
743 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
744 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
745 MIN_NUM_REGIONS,
746 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
747 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
748 }
749 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
750 err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
751 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
752 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
753 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
754 }
755 if (ShenandoahMinRegionSize < MinTLABSize) {
756 err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
757 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
758 byte_size_in_proper_unit(MinTLABSize), proper_unit_for_byte_size(MinTLABSize));
759 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
760 }
761 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
762 err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
763 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
764 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
765 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
766 }
767 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
768 err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
769 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
770 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
771 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
772 }
773
774 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
775 // for usual heap sizes. Do not depend on initial_heap_size here.
776 region_size = max_heap_size / ShenandoahTargetNumRegions;
777
778 // Now make sure that we don't go over or under our limits.
779 region_size = MAX2(ShenandoahMinRegionSize, region_size);
780 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
781
782 } else {
783 if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
784 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
785 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
786 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
787 MIN_NUM_REGIONS,
788 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
789 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
790 }
791 if (ShenandoahRegionSize < ShenandoahMinRegionSize) {
792 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
793 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
794 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
795 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
796 }
797 if (ShenandoahRegionSize > ShenandoahMaxRegionSize) {
798 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
799 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
800 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
801 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
802 }
803 region_size = ShenandoahRegionSize;
804 }
805
806 // Make sure region size and heap size are page aligned.
807 // If large pages are used, we ensure that region size is aligned to large page size if
808 // heap size is large enough to accommodate minimal number of regions. Otherwise, we align
809 // region size to regular page size.
810
811 // Figure out page size to use, and aligns up heap to page size
812 size_t page_size = os::vm_page_size();
813 if (UseLargePages) {
814 size_t large_page_size = os::large_page_size();
815 max_heap_size = align_up(max_heap_size, large_page_size);
816 if ((max_heap_size / align_up(region_size, large_page_size)) >= MIN_NUM_REGIONS) {
817 page_size = large_page_size;
818 } else {
819 // Should have been checked during argument initialization
820 assert(!ShenandoahUncommit, "Uncommit requires region size aligns to large page size");
821 }
822 } else {
823 max_heap_size = align_up(max_heap_size, page_size);
824 }
825
826 // Align region size to page size
827 region_size = align_up(region_size, page_size);
828
829 int region_size_log = log2i(region_size);
830 // Recalculate the region size to make sure it's a power of
831 // 2. This means that region_size is the largest power of 2 that's
832 // <= what we've calculated so far.
833 region_size = size_t(1) << region_size_log;
834
835 // Now, set up the globals.
836 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
837 RegionSizeBytesShift = (size_t)region_size_log;
838
839 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
840 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
841
842 guarantee(RegionSizeBytes == 0, "we should only set it once");
843 RegionSizeBytes = region_size;
844 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
845 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
846
847 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
848 RegionSizeWordsMask = RegionSizeWords - 1;
849
850 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
851 RegionSizeBytesMask = RegionSizeBytes - 1;
852
853 guarantee(RegionCount == 0, "we should only set it once");
854 RegionCount = align_up(max_heap_size, RegionSizeBytes) / RegionSizeBytes;
855 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
856
857 guarantee(HumongousThresholdWords == 0, "we should only set it once");
858 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
859 HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
860 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
861
862 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
863 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
864 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
865
866 // The rationale for trimming the TLAB sizes has to do with the raciness in
867 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
868 // about next free size, gets the answer for region #N, goes away for a while, then
869 // tries to allocate in region #N, and fail because some other thread have claimed part
870 // of the region #N, and then the freeset allocation code has to retire the region #N,
871 // before moving the allocation to region #N+1.
872 //
873 // The worst case realizes when "answer" is "region size", which means it could
874 // prematurely retire an entire region. Having smaller TLABs does not fix that
875 // completely, but reduces the probability of too wasteful region retirement.
876 // With current divisor, we will waste no more than 1/8 of region size in the worst
877 // case. This also has a secondary effect on collection set selection: even under
878 // the race, the regions would be at least 7/8 used, which allows relying on
879 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
880 // below the garbage threshold that would never be considered for collection.
881 //
882 // The whole thing is mitigated if Elastic TLABs are enabled.
883 //
884 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
885 MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
886 MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
887
888 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
889 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
890 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
891
892 return max_heap_size;
893 }
894
895 void ShenandoahHeapRegion::do_commit() {
896 ShenandoahHeap* heap = ShenandoahHeap::heap();
897 if (!heap->is_heap_region_special() && !os::commit_memory((char *) bottom(), RegionSizeBytes, false)) {
898 report_java_out_of_memory("Unable to commit region");
899 }
900 if (!heap->commit_bitmap_slice(this)) {
901 report_java_out_of_memory("Unable to commit bitmaps for region");
902 }
903 if (AlwaysPreTouch) {
904 os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size());
905 }
906 heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
907 }
908
909 void ShenandoahHeapRegion::do_uncommit() {
910 ShenandoahHeap* heap = ShenandoahHeap::heap();
911 if (!heap->is_heap_region_special() && !os::uncommit_memory((char *) bottom(), RegionSizeBytes)) {
912 report_java_out_of_memory("Unable to uncommit region");
913 }
914 if (!heap->uncommit_bitmap_slice(this)) {
915 report_java_out_of_memory("Unable to uncommit bitmaps for region");
916 }
917 heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
918 }
919
920 void ShenandoahHeapRegion::set_state(RegionState to) {
921 EventShenandoahHeapRegionStateChange evt;
922 if (evt.should_commit()){
923 evt.set_index((unsigned) index());
924 evt.set_start((uintptr_t)bottom());
925 evt.set_used(used());
926 evt.set_from(_state);
927 evt.set_to(to);
928 evt.commit();
929 }
930 _state = to;
931 }
932
933 void ShenandoahHeapRegion::record_pin() {
934 Atomic::add(&_critical_pins, (size_t)1);
935 }
936
937 void ShenandoahHeapRegion::record_unpin() {
938 assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
939 Atomic::sub(&_critical_pins, (size_t)1);
940 }
941
942 size_t ShenandoahHeapRegion::pin_count() const {
943 return Atomic::load(&_critical_pins);
944 }
945
946 void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) {
947 ShenandoahHeap* heap = ShenandoahHeap::heap();
948
949 ShenandoahAffiliation region_affiliation = heap->region_affiliation(this);
950 {
951 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
952 log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
953 ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
954 index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation),
955 p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this)));
956 }
957
958 #ifdef ASSERT
959 {
960 // During full gc, heap->complete_marking_context() is not valid, may equal nullptr.
961 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
962 size_t idx = this->index();
963 HeapWord* top_bitmap = ctx->top_bitmap(this);
964
965 assert(ctx->is_bitmap_clear_range(top_bitmap, _end),
966 "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx,
967 p2i(top_bitmap), p2i(_end));
968 }
969 #endif
970
971 if (region_affiliation == new_affiliation) {
972 return;
973 }
974
975 if (!heap->mode()->is_generational()) {
976 log_trace(gc)("Changing affiliation of region %zu from %s to %s",
977 index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation));
978 heap->set_affiliation(this, new_affiliation);
979 return;
980 }
981
982 switch (new_affiliation) {
983 case FREE:
984 assert(!has_live(), "Free region should not have live data");
985 break;
986 case YOUNG_GENERATION:
987 reset_age();
988 break;
989 case OLD_GENERATION:
990 // TODO: should we reset_age() for OLD as well? Examine invocations of set_affiliation(). Some contexts redundantly
991 // invoke reset_age().
992 break;
993 default:
994 ShouldNotReachHere();
995 return;
996 }
997 heap->set_affiliation(this, new_affiliation);
998 }
999
1000 // When we promote a region in place, we can continue to use the established marking context to guide subsequent remembered
1001 // set scans of this region's content. The region will be coalesced and filled prior to the next old-gen marking effort.
1002 // We identify the entirety of the region as DIRTY to force the next remembered set scan to identify the "interesting poitners"
1003 // contained herein.
1004 void ShenandoahHeapRegion::promote_in_place() {
1005 ShenandoahHeap* heap = ShenandoahHeap::heap();
1006 ShenandoahMarkingContext* marking_context = heap->marking_context();
1007 HeapWord* tams = marking_context->top_at_mark_start(this);
1008 assert(heap->active_generation()->is_mark_complete(), "sanity");
1009 assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot promote in place during old marking");
1010 assert(is_young(), "Only young regions can be promoted");
1011 assert(is_regular(), "Use different service to promote humongous regions");
1012 assert(age() >= heap->age_census()->tenuring_threshold(), "Only promote regions that are sufficiently aged");
1013
1014 ShenandoahOldGeneration* old_gen = heap->old_generation();
1015 ShenandoahYoungGeneration* young_gen = heap->young_generation();
1016 size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
1017
1018 assert(get_top_before_promote() == tams, "Cannot promote regions in place if top has advanced beyond TAMS");
1019
1020 // Rebuild the remembered set information and mark the entire range as DIRTY. We do NOT scan the content of this
1021 // range to determine which cards need to be DIRTY. That would force us to scan the region twice, once now, and
1022 // once during the subsequent remembered set scan. Instead, we blindly (conservatively) mark everything as DIRTY
1023 // now and then sort out the CLEAN pages during the next remembered set scan.
1024 //
1025 // Rebuilding the remembered set consists of clearing all object registrations (reset_object_range()) here,
1026 // then registering every live object and every coalesced range of free objects in the loop that follows.
1027 heap->card_scan()->reset_object_range(bottom(), end());
1028 heap->card_scan()->mark_range_as_dirty(bottom(), get_top_before_promote() - bottom());
1029
1030 // TODO: use an existing coalesce-and-fill function rather than replicating the code here.
1031 HeapWord* obj_addr = bottom();
1032 while (obj_addr < tams) {
1033 oop obj = cast_to_oop(obj_addr);
1034 if (marking_context->is_marked(obj)) {
1035 assert(obj->klass() != nullptr, "klass should not be NULL");
1036 // This thread is responsible for registering all objects in this region. No need for lock.
1037 heap->card_scan()->register_object_without_lock(obj_addr);
1038 obj_addr += obj->size();
1039 } else {
1040 HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, tams);
1041 assert(next_marked_obj <= tams, "next marked object cannot exceed tams");
1042 size_t fill_size = next_marked_obj - obj_addr;
1043 assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size");
1044 ShenandoahHeap::fill_with_object(obj_addr, fill_size);
1045 heap->card_scan()->register_object_without_lock(obj_addr);
1046 obj_addr = next_marked_obj;
1047 }
1048 }
1049 // We do not need to scan above TAMS because restored top equals tams
1050 assert(obj_addr == tams, "Expect loop to terminate when obj_addr equals tams");
1051
1052 {
1053 ShenandoahHeapLocker locker(heap->lock());
1054
1055 HeapWord* update_watermark = get_update_watermark();
1056
1057 // Now that this region is affiliated with old, we can allow it to receive allocations, though it may not be in the
1058 // is_collector_free range.
1059 restore_top_before_promote();
1060
1061 size_t region_capacity = free();
1062 size_t region_used = used();
1063
1064 // The update_watermark was likely established while we had the artificially high value of top. Make it sane now.
1065 assert(update_watermark >= top(), "original top cannot exceed preserved update_watermark");
1066 set_update_watermark(top());
1067
1068 // Unconditionally transfer one region from young to old to represent the newly promoted region.
1069 // This expands old and shrinks new by the size of one region. Strictly, we do not "need" to expand old
1070 // if there are already enough unaffiliated regions in old to account for this newly promoted region.
1071 // However, if we do not transfer the capacities, we end up reducing the amount of memory that would have
1072 // otherwise been available to hold old evacuations, because old available is max_capacity - used and now
1073 // we would be trading a fully empty region for a partially used region.
1074
1075 young_gen->decrease_used(region_used);
1076 young_gen->decrement_affiliated_region_count();
1077
1078 // transfer_to_old() increases capacity of old and decreases capacity of young
1079 heap->generation_sizer()->force_transfer_to_old(1);
1080 set_affiliation(OLD_GENERATION);
1081
1082 old_gen->increment_affiliated_region_count();
1083 old_gen->increase_used(region_used);
1084
1085 // add_old_collector_free_region() increases promoted_reserve() if available space exceeds PLAB::min_size()
1086 heap->free_set()->add_old_collector_free_region(this);
1087 }
1088 }
1089
1090 void ShenandoahHeapRegion::promote_humongous() {
1091 ShenandoahHeap* heap = ShenandoahHeap::heap();
1092 ShenandoahMarkingContext* marking_context = heap->marking_context();
1093 assert(heap->active_generation()->is_mark_complete(), "sanity");
1094 assert(is_young(), "Only young regions can be promoted");
1095 assert(is_humongous_start(), "Should not promote humongous continuation in isolation");
1096 assert(age() >= heap->age_census()->tenuring_threshold(), "Only promote regions that are sufficiently aged");
1097
1098 ShenandoahGeneration* old_generation = heap->old_generation();
1099 ShenandoahGeneration* young_generation = heap->young_generation();
1100
1101 oop obj = cast_to_oop(bottom());
1102 assert(marking_context->is_marked(obj), "promoted humongous object should be alive");
1103
1104 // TODO: Consider not promoting humongous objects that represent primitive arrays. Leaving a primitive array
1105 // (obj->is_typeArray()) in young-gen is harmless because these objects are never relocated and they are not
1106 // scanned. Leaving primitive arrays in young-gen memory allows their memory to be reclaimed more quickly when
1107 // it becomes garbage. Better to not make this change until sizes of young-gen and old-gen are completely
1108 // adaptive, as leaving primitive arrays in young-gen might be perceived as an "astonishing result" by someone
1109 // has carefully analyzed the required sizes of an application's young-gen and old-gen.
1110 size_t used_bytes = obj->size() * HeapWordSize;
1111 size_t spanned_regions = ShenandoahHeapRegion::required_regions(used_bytes);
1112 size_t humongous_waste = spanned_regions * ShenandoahHeapRegion::region_size_bytes() - obj->size() * HeapWordSize;
1113 size_t index_limit = index() + spanned_regions;
1114 {
1115 // We need to grab the heap lock in order to avoid a race when changing the affiliations of spanned_regions from
1116 // young to old.
1117 ShenandoahHeapLocker locker(heap->lock());
1118
1119 // We promote humongous objects unconditionally, without checking for availability. We adjust
1120 // usage totals, including humongous waste, after evacuation is done.
1121 log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions);
1122
1123 young_generation->decrease_used(used_bytes);
1124 young_generation->decrease_humongous_waste(humongous_waste);
1125 young_generation->decrease_affiliated_region_count(spanned_regions);
1126
1127 // transfer_to_old() increases capacity of old and decreases capacity of young
1128 heap->generation_sizer()->force_transfer_to_old(spanned_regions);
1129
1130 // For this region and each humongous continuation region spanned by this humongous object, change
1131 // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory
1132 // in the last humongous region that is not spanned by obj is currently not used.
1133 for (size_t i = index(); i < index_limit; i++) {
1134 ShenandoahHeapRegion* r = heap->get_region(i);
1135 log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT,
1136 r->index(), p2i(r->bottom()), p2i(r->top()));
1137 // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here
1138 r->set_affiliation(OLD_GENERATION);
1139 }
1140
1141 old_generation->increase_affiliated_region_count(spanned_regions);
1142 old_generation->increase_used(used_bytes);
1143 old_generation->increase_humongous_waste(humongous_waste);
1144 }
1145
1146 // Since this region may have served previously as OLD, it may hold obsolete object range info.
1147 heap->card_scan()->reset_object_range(bottom(), bottom() + spanned_regions * ShenandoahHeapRegion::region_size_words());
1148 // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation.
1149 heap->card_scan()->register_object_without_lock(bottom());
1150
1151 if (obj->is_typeArray()) {
1152 // Primitive arrays don't need to be scanned.
1153 log_debug(gc)("Clean cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
1154 index(), p2i(bottom()), p2i(bottom() + obj->size()));
1155 heap->card_scan()->mark_range_as_clean(bottom(), obj->size());
1156 } else {
1157 log_debug(gc)("Dirty cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
1158 index(), p2i(bottom()), p2i(bottom() + obj->size()));
1159 heap->card_scan()->mark_range_as_dirty(bottom(), obj->size());
1160 }
1161 }
1162
1163 void ShenandoahHeapRegion::decrement_humongous_waste() const {
1164 assert(is_humongous(), "Should only use this for humongous regions");
1165 size_t waste_bytes = free();
1166 if (waste_bytes > 0) {
1167 ShenandoahHeap* heap = ShenandoahHeap::heap();
1168 ShenandoahGeneration* generation = heap->generation_for(affiliation());
1169 heap->decrease_humongous_waste(generation, waste_bytes);
1170 }
1171 }