1 /*
2 * Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "gc/g1/g1Allocator.inline.hpp"
26 #include "gc/g1/g1AllocRegion.inline.hpp"
27 #include "gc/g1/g1CollectedHeap.inline.hpp"
28 #include "gc/g1/g1EvacInfo.hpp"
29 #include "gc/g1/g1EvacStats.inline.hpp"
30 #include "gc/g1/g1HeapRegion.inline.hpp"
31 #include "gc/g1/g1HeapRegionPrinter.hpp"
32 #include "gc/g1/g1HeapRegionSet.inline.hpp"
33 #include "gc/g1/g1HeapRegionType.hpp"
34 #include "gc/g1/g1NUMA.hpp"
35 #include "gc/g1/g1Policy.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "runtime/mutexLocker.hpp"
38 #include "utilities/align.hpp"
39
40 G1Allocator::G1Allocator(G1CollectedHeap* heap) :
41 _g1h(heap),
42 _numa(heap->numa()),
43 _survivor_is_full(false),
44 _old_is_full(false),
45 _num_alloc_regions(_numa->num_active_nodes()),
46 _mutator_alloc_regions(nullptr),
47 _survivor_gc_alloc_regions(nullptr),
48 _old_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Old)),
49 _retained_old_gc_alloc_region(nullptr) {
50
51 _mutator_alloc_regions = NEW_C_HEAP_ARRAY(MutatorAllocRegion, _num_alloc_regions, mtGC);
52 _survivor_gc_alloc_regions = NEW_C_HEAP_ARRAY(SurvivorGCAllocRegion, _num_alloc_regions, mtGC);
53 G1EvacStats* stat = heap->alloc_buffer_stats(G1HeapRegionAttr::Young);
54
55 for (uint i = 0; i < _num_alloc_regions; i++) {
56 ::new(_mutator_alloc_regions + i) MutatorAllocRegion(i);
57 ::new(_survivor_gc_alloc_regions + i) SurvivorGCAllocRegion(stat, i);
58 }
59 }
60
61 G1Allocator::~G1Allocator() {
62 for (uint i = 0; i < _num_alloc_regions; i++) {
63 _mutator_alloc_regions[i].~MutatorAllocRegion();
64 _survivor_gc_alloc_regions[i].~SurvivorGCAllocRegion();
65 }
66 FREE_C_HEAP_ARRAY(_mutator_alloc_regions);
67 FREE_C_HEAP_ARRAY(_survivor_gc_alloc_regions);
68 }
69
70 #ifdef ASSERT
71 bool G1Allocator::has_mutator_alloc_region() {
72 uint node_index = current_node_index();
73 return mutator_alloc_region(node_index)->get() != nullptr;
74 }
75 #endif
76
77 void G1Allocator::init_mutator_alloc_regions() {
78 for (uint i = 0; i < _num_alloc_regions; i++) {
79 assert(mutator_alloc_region(i)->get() == nullptr, "pre-condition");
80 mutator_alloc_region(i)->init();
81 }
82 }
83
84 void G1Allocator::release_mutator_alloc_regions() {
85 for (uint i = 0; i < _num_alloc_regions; i++) {
86 mutator_alloc_region(i)->release();
87 assert(mutator_alloc_region(i)->get() == nullptr, "post-condition");
88 }
89 }
90
91 bool G1Allocator::is_retained_old_region(G1HeapRegion* hr) {
92 return _retained_old_gc_alloc_region == hr;
93 }
94
95 void G1Allocator::reuse_retained_old_region(G1EvacInfo* evacuation_info,
96 OldGCAllocRegion* old,
97 G1HeapRegion** retained_old) {
98 G1HeapRegion* retained_region = *retained_old;
99 *retained_old = nullptr;
100
101 // We will discard the current GC alloc region if:
102 // a) it's in the collection set (it can happen!),
103 // b) it's already full (no point in using it),
104 // c) it's empty (this means that it was emptied during
105 // a cleanup and it should be on the free list now), or
106 // d) it's humongous (this means that it was emptied
107 // during a cleanup and was added to the free list, but
108 // has been subsequently used to allocate a humongous
109 // object that may be less than the region size).
110 if (retained_region != nullptr &&
111 !retained_region->in_collection_set() &&
112 !(retained_region->top() == retained_region->end()) &&
113 !retained_region->is_empty() &&
114 !retained_region->is_humongous()) {
115 // The retained region was added to the old region set when it was
116 // retired. We have to remove it now, since we don't allow regions
117 // we allocate to in the region sets. We'll re-add it later, when
118 // it's retired again.
119 _g1h->old_set_remove(retained_region);
120 old->reuse(retained_region);
121 G1HeapRegionPrinter::reuse(retained_region);
122 evacuation_info->set_alloc_regions_used_before(retained_region->used());
123 }
124 }
125
126 size_t G1Allocator::free_bytes_in_retained_old_region() const {
127 if (_retained_old_gc_alloc_region == nullptr) {
128 return 0;
129 } else {
130 return _retained_old_gc_alloc_region->free();
131 }
132 }
133
134 void G1Allocator::init_gc_alloc_regions(G1EvacInfo* evacuation_info) {
135 assert_at_safepoint_on_vm_thread();
136
137 _survivor_is_full = false;
138 _old_is_full = false;
139
140 for (uint i = 0; i < _num_alloc_regions; i++) {
141 survivor_gc_alloc_region(i)->init();
142 }
143
144 _old_gc_alloc_region.init();
145 reuse_retained_old_region(evacuation_info,
146 &_old_gc_alloc_region,
147 &_retained_old_gc_alloc_region);
148 }
149
150 void G1Allocator::release_gc_alloc_regions(G1EvacInfo* evacuation_info) {
151 uint survivor_region_count = 0;
152 for (uint node_index = 0; node_index < _num_alloc_regions; node_index++) {
153 survivor_region_count += survivor_gc_alloc_region(node_index)->count();
154 survivor_gc_alloc_region(node_index)->release();
155 }
156 evacuation_info->set_allocation_regions(survivor_region_count +
157 old_gc_alloc_region()->count());
158
159 // If we have an old GC alloc region to release, we'll save it in
160 // _retained_old_gc_alloc_region. If we don't
161 // _retained_old_gc_alloc_region will become null. This is what we
162 // want either way so no reason to check explicitly for either
163 // condition.
164 _retained_old_gc_alloc_region = old_gc_alloc_region()->release();
165 }
166
167 void G1Allocator::abandon_gc_alloc_regions() {
168 for (uint i = 0; i < _num_alloc_regions; i++) {
169 assert(survivor_gc_alloc_region(i)->get() == nullptr, "pre-condition");
170 }
171 assert(old_gc_alloc_region()->get() == nullptr, "pre-condition");
172 _retained_old_gc_alloc_region = nullptr;
173 }
174
175 bool G1Allocator::survivor_is_full() const {
176 return _survivor_is_full;
177 }
178
179 bool G1Allocator::old_is_full() const {
180 return _old_is_full;
181 }
182
183 void G1Allocator::set_survivor_full() {
184 _survivor_is_full = true;
185 }
186
187 void G1Allocator::set_old_full() {
188 _old_is_full = true;
189 }
190
191 size_t G1Allocator::unsafe_max_tlab_alloc() {
192 // Return the remaining space in the cur alloc region, but not less than
193 // the min TLAB size.
194
195 // Also, this value can be at most the humongous object threshold,
196 // since we can't allow tlabs to grow big enough to accommodate
197 // humongous objects.
198
199 uint node_index = current_node_index();
200 G1HeapRegion* hr = mutator_alloc_region(node_index)->get();
201 size_t max_tlab = _g1h->max_tlab_size() * wordSize;
202
203 if (hr == nullptr || hr->free() < MinTLABSize) {
204 // The next TLAB allocation will most probably happen in a new region,
205 // therefore we can attempt to allocate the maximum allowed TLAB size.
206 return max_tlab;
207 }
208
209 return MIN2(hr->free(), max_tlab);
210 }
211
212 size_t G1Allocator::used_in_alloc_regions() {
213 assert(Heap_lock->owner() != nullptr, "Should be owned on this thread's behalf.");
214 size_t used = 0;
215 for (uint i = 0; i < _num_alloc_regions; i++) {
216 used += mutator_alloc_region(i)->used_in_alloc_regions();
217 }
218 return used;
219 }
220
221
222 HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
223 uint node_index,
224 size_t word_size) {
225 size_t temp = 0;
226 HeapWord* result = par_allocate_during_gc(dest, node_index, word_size, word_size, &temp);
227 assert(result == nullptr || temp == word_size,
228 "Requested %zu words, but got %zu at " PTR_FORMAT,
229 word_size, temp, p2i(result));
230 return result;
231 }
232
233 HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
234 uint node_index,
235 size_t min_word_size,
236 size_t desired_word_size,
237 size_t* actual_word_size) {
238 switch (dest.type()) {
239 case G1HeapRegionAttr::Young:
240 return survivor_attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
241 case G1HeapRegionAttr::Old:
242 return old_attempt_allocation(min_word_size, desired_word_size, actual_word_size);
243 default:
244 ShouldNotReachHere();
245 return nullptr; // Keep some compilers happy
246 }
247 }
248
249 #ifdef ASSERT
250 void G1Allocator::assert_not_humongous(size_t word_size) {
251 // With CompactObjectHeaders, objects can expand during copy to accomodate hashcode.
252 // It's possible this expansion crosses the humongous threshold. In this case, we allow
253 // that and just treat it as not humongous.
254 size_t pre_expansion_size = UseCompactObjectHeaders ? word_size - 1 : word_size;
255 assert(!_g1h->is_humongous(pre_expansion_size),
256 "we should not be seeing humongous-size allocations in this path");
257 }
258 #endif
259
260 HeapWord* G1Allocator::survivor_attempt_allocation(uint node_index,
261 size_t min_word_size,
262 size_t desired_word_size,
263 size_t* actual_word_size) {
264 assert_not_humongous(desired_word_size);
265
266 HeapWord* result = survivor_gc_alloc_region(node_index)->attempt_allocation(min_word_size,
267 desired_word_size,
268 actual_word_size);
269 if (result == nullptr && !survivor_is_full()) {
270 MutexLocker x(G1FreeList_lock, Mutex::_no_safepoint_check_flag);
271 // Multiple threads may have queued at the FreeList_lock above after checking whether there
272 // actually is still memory available. Redo the check under the lock to avoid unnecessary work;
273 // the memory may have been used up as the threads waited to acquire the lock.
274 if (!survivor_is_full()) {
275 result = survivor_gc_alloc_region(node_index)->attempt_allocation_locked(min_word_size,
276 desired_word_size,
277 actual_word_size);
278 if (result == nullptr) {
279 set_survivor_full();
280 }
281 }
282 }
283 return result;
284 }
285
286 HeapWord* G1Allocator::old_attempt_allocation(size_t min_word_size,
287 size_t desired_word_size,
288 size_t* actual_word_size) {
289 assert_not_humongous(desired_word_size);
290
291 HeapWord* result = old_gc_alloc_region()->attempt_allocation(min_word_size,
292 desired_word_size,
293 actual_word_size);
294 if (result == nullptr && !old_is_full()) {
295 MutexLocker x(G1FreeList_lock, Mutex::_no_safepoint_check_flag);
296 // Multiple threads may have queued at the FreeList_lock above after checking whether there
297 // actually is still memory available. Redo the check under the lock to avoid unnecessary work;
298 // the memory may have been used up as the threads waited to acquire the lock.
299 if (!old_is_full()) {
300 result = old_gc_alloc_region()->attempt_allocation_locked(min_word_size,
301 desired_word_size,
302 actual_word_size);
303 if (result == nullptr) {
304 set_old_full();
305 }
306 }
307 }
308 return result;
309 }
310
311 G1PLABAllocator::PLABData::PLABData() :
312 _alloc_buffer(nullptr),
313 _direct_allocated(0),
314 _num_plab_fills(0),
315 _num_direct_allocations(0),
316 _plab_fill_counter(0),
317 _cur_desired_plab_size(0),
318 _num_alloc_buffers(0) { }
319
320 G1PLABAllocator::PLABData::~PLABData() {
321 if (_alloc_buffer == nullptr) {
322 return;
323 }
324 for (uint node_index = 0; node_index < _num_alloc_buffers; node_index++) {
325 delete _alloc_buffer[node_index];
326 }
327 FREE_C_HEAP_ARRAY(_alloc_buffer);
328 }
329
330 void G1PLABAllocator::PLABData::initialize(uint num_alloc_buffers, size_t desired_plab_size, size_t tolerated_refills) {
331 _num_alloc_buffers = num_alloc_buffers;
332 _alloc_buffer = NEW_C_HEAP_ARRAY(PLAB*, _num_alloc_buffers, mtGC);
333
334 for (uint node_index = 0; node_index < _num_alloc_buffers; node_index++) {
335 _alloc_buffer[node_index] = new PLAB(desired_plab_size);
336 }
337
338 _plab_fill_counter = tolerated_refills;
339 _cur_desired_plab_size = desired_plab_size;
340 }
341
342 void G1PLABAllocator::PLABData::notify_plab_refill(size_t tolerated_refills, size_t next_plab_size) {
343 _num_plab_fills++;
344 if (should_boost()) {
345 _plab_fill_counter = tolerated_refills;
346 _cur_desired_plab_size = next_plab_size;
347 } else {
348 _plab_fill_counter--;
349 }
350 }
351
352 G1PLABAllocator::G1PLABAllocator(G1Allocator* allocator) :
353 _g1h(G1CollectedHeap::heap()),
354 _allocator(allocator) {
355
356 if (ResizePLAB) {
357 // See G1EvacStats::compute_desired_plab_sz for the reasoning why this is the
358 // expected number of refills.
359 double const ExpectedNumberOfRefills = (100 - G1LastPLABAverageOccupancy) / TargetPLABWastePct;
360 // Add some padding to the threshold to not boost exactly when the targeted refills
361 // were reached.
362 // E.g. due to limitation of PLAB size to non-humongous objects and region boundaries
363 // a thread may experience more refills than expected. Keeping the PLAB waste low
364 // is the main goal, so being a bit conservative is better.
365 double const PadFactor = 1.5;
366 _tolerated_refills = MAX2(ExpectedNumberOfRefills, 1.0) * PadFactor;
367 } else {
368 // Make the tolerated refills a huge number.
369 _tolerated_refills = SIZE_MAX;
370 }
371 // The initial PLAB refill should not count, hence the +1 for the first boost.
372 size_t initial_tolerated_refills = ResizePLAB ? _tolerated_refills + 1 : _tolerated_refills;
373 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
374 _dest_data[state].initialize(alloc_buffers_length(state), _g1h->desired_plab_sz(state), initial_tolerated_refills);
375 }
376 }
377
378 bool G1PLABAllocator::may_throw_away_buffer(size_t const words_remaining, size_t const buffer_size) const {
379 return (words_remaining * 100 < buffer_size * ParallelGCBufferWastePct);
380 }
381
382 HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
383 size_t word_sz,
384 bool* plab_refill_failed,
385 uint node_index) {
386 PLAB* alloc_buf = alloc_buffer(dest, node_index);
387 size_t words_remaining = alloc_buf->words_remaining();
388 assert(words_remaining < word_sz, "precondition");
389
390 size_t plab_word_size = plab_size(dest.type());
391 size_t next_plab_word_size = plab_word_size;
392
393 PLABData* plab_data = &_dest_data[dest.type()];
394
395 if (plab_data->should_boost()) {
396 next_plab_word_size = _g1h->clamp_plab_size(next_plab_word_size * 2);
397 }
398
399 size_t required_in_plab = PLAB::size_required_for_allocation(word_sz);
400
401 // Only get a new PLAB if the allocation fits into the to-be-allocated PLAB and
402 // retiring the current PLAB would not waste more than ParallelGCBufferWastePct
403 // in the current PLAB. Boosting the PLAB also increasingly allows more waste to occur.
404 if ((required_in_plab <= next_plab_word_size) &&
405 may_throw_away_buffer(words_remaining, plab_word_size)) {
406
407 alloc_buf->retire();
408
409 plab_data->notify_plab_refill(_tolerated_refills, next_plab_word_size);
410 plab_word_size = next_plab_word_size;
411
412 size_t actual_plab_size = 0;
413 HeapWord* buf = _allocator->par_allocate_during_gc(dest,
414 node_index,
415 required_in_plab,
416 plab_word_size,
417 &actual_plab_size);
418
419 assert(buf == nullptr || ((actual_plab_size >= required_in_plab) && (actual_plab_size <= plab_word_size)),
420 "Requested at minimum %zu, desired %zu words, but got %zu at " PTR_FORMAT,
421 required_in_plab, plab_word_size, actual_plab_size, p2i(buf));
422
423 if (buf != nullptr) {
424 alloc_buf->set_buf(buf, actual_plab_size);
425
426 HeapWord* const obj = alloc_buf->allocate(word_sz);
427 assert(obj != nullptr, "PLAB should have been big enough, tried to allocate "
428 "%zu requiring %zu PLAB size %zu",
429 word_sz, required_in_plab, plab_word_size);
430 return obj;
431 }
432 // Otherwise.
433 *plab_refill_failed = true;
434 }
435 // Try direct allocation.
436 HeapWord* result = _allocator->par_allocate_during_gc(dest, node_index, word_sz);
437 if (result != nullptr) {
438 plab_data->_direct_allocated += word_sz;
439 plab_data->_num_direct_allocations++;
440 }
441 return result;
442 }
443
444 void G1PLABAllocator::undo_allocation(G1HeapRegionAttr dest, HeapWord* obj, size_t word_sz, uint node_index) {
445 alloc_buffer(dest, node_index)->undo_allocation(obj, word_sz);
446 }
447
448 void G1PLABAllocator::flush_and_retire_stats(uint num_workers) {
449 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
450 G1EvacStats* stats = _g1h->alloc_buffer_stats(state);
451 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
452 PLAB* const buf = alloc_buffer(state, node_index);
453 if (buf != nullptr) {
454 buf->flush_and_retire_stats(stats);
455 }
456 }
457 PLABData* plab_data = &_dest_data[state];
458 stats->add_num_plab_filled(plab_data->_num_plab_fills);
459 stats->add_direct_allocated(plab_data->_direct_allocated);
460 stats->add_num_direct_allocated(plab_data->_num_direct_allocations);
461 }
462
463 log_trace(gc, plab)("PLAB boost: Young %zu -> %zu refills %zu (tolerated %zu) Old %zu -> %zu refills %zu (tolerated %zu)",
464 _g1h->alloc_buffer_stats(G1HeapRegionAttr::Young)->desired_plab_size(num_workers),
465 plab_size(G1HeapRegionAttr::Young),
466 _dest_data[G1HeapRegionAttr::Young]._num_plab_fills,
467 _tolerated_refills,
468 _g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->desired_plab_size(num_workers),
469 plab_size(G1HeapRegionAttr::Old),
470 _dest_data[G1HeapRegionAttr::Old]._num_plab_fills,
471 _tolerated_refills);
472 }
473
474 size_t G1PLABAllocator::waste() const {
475 size_t result = 0;
476 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
477 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
478 PLAB* const buf = alloc_buffer(state, node_index);
479 if (buf != nullptr) {
480 result += buf->waste();
481 }
482 }
483 }
484 return result;
485 }
486
487 size_t G1PLABAllocator::plab_size(G1HeapRegionAttr which) const {
488 return _dest_data[which.type()]._cur_desired_plab_size;
489 }
490
491 size_t G1PLABAllocator::undo_waste() const {
492 size_t result = 0;
493 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
494 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
495 PLAB* const buf = alloc_buffer(state, node_index);
496 if (buf != nullptr) {
497 result += buf->undo_waste();
498 }
499 }
500 }
501 return result;
502 }