1 /*
2 * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "gc/g1/g1Allocator.inline.hpp"
26 #include "gc/g1/g1AllocRegion.inline.hpp"
27 #include "gc/g1/g1CollectedHeap.inline.hpp"
28 #include "gc/g1/g1EvacInfo.hpp"
29 #include "gc/g1/g1EvacStats.inline.hpp"
30 #include "gc/g1/g1HeapRegion.inline.hpp"
31 #include "gc/g1/g1HeapRegionPrinter.hpp"
32 #include "gc/g1/g1HeapRegionSet.inline.hpp"
33 #include "gc/g1/g1HeapRegionType.hpp"
34 #include "gc/g1/g1NUMA.hpp"
35 #include "gc/g1/g1Policy.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "runtime/mutexLocker.hpp"
38 #include "utilities/align.hpp"
39
40 G1Allocator::G1Allocator(G1CollectedHeap* heap) :
41 _g1h(heap),
42 _numa(heap->numa()),
43 _survivor_is_full(false),
44 _old_is_full(false),
45 _num_alloc_regions(_numa->num_active_nodes()),
46 _mutator_alloc_regions(nullptr),
47 _survivor_gc_alloc_regions(nullptr),
48 _old_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Old)),
49 _retained_old_gc_alloc_region(nullptr) {
50
51 _mutator_alloc_regions = NEW_C_HEAP_ARRAY(MutatorAllocRegion, _num_alloc_regions, mtGC);
52 _survivor_gc_alloc_regions = NEW_C_HEAP_ARRAY(SurvivorGCAllocRegion, _num_alloc_regions, mtGC);
53 G1EvacStats* stat = heap->alloc_buffer_stats(G1HeapRegionAttr::Young);
54
55 for (uint i = 0; i < _num_alloc_regions; i++) {
56 ::new(_mutator_alloc_regions + i) MutatorAllocRegion(i);
57 ::new(_survivor_gc_alloc_regions + i) SurvivorGCAllocRegion(stat, i);
58 }
59 }
60
61 G1Allocator::~G1Allocator() {
62 for (uint i = 0; i < _num_alloc_regions; i++) {
63 _mutator_alloc_regions[i].~MutatorAllocRegion();
64 _survivor_gc_alloc_regions[i].~SurvivorGCAllocRegion();
65 }
66 FREE_C_HEAP_ARRAY(_mutator_alloc_regions);
67 FREE_C_HEAP_ARRAY(_survivor_gc_alloc_regions);
68 }
69
70 #ifdef ASSERT
71 bool G1Allocator::has_mutator_alloc_region() {
72 uint node_index = current_node_index();
73 return mutator_alloc_region(node_index)->get() != nullptr;
74 }
75 #endif
76
77 void G1Allocator::init_mutator_alloc_regions() {
78 for (uint i = 0; i < _num_alloc_regions; i++) {
79 assert(mutator_alloc_region(i)->get() == nullptr, "pre-condition");
80 mutator_alloc_region(i)->init();
81 }
82 }
83
84 void G1Allocator::release_mutator_alloc_regions() {
85 for (uint i = 0; i < _num_alloc_regions; i++) {
86 mutator_alloc_region(i)->release();
87 assert(mutator_alloc_region(i)->get() == nullptr, "post-condition");
88 }
89 }
90
91 bool G1Allocator::is_retained_old_region(G1HeapRegion* hr) {
92 return _retained_old_gc_alloc_region == hr;
93 }
94
95 void G1Allocator::reuse_retained_old_region(G1EvacInfo* evacuation_info,
96 OldGCAllocRegion* old,
97 G1HeapRegion** retained_old) {
98 G1HeapRegion* retained_region = *retained_old;
99 *retained_old = nullptr;
100
101 // We will discard the current GC alloc region if:
102 // a) it's in the collection set (it can happen!),
103 // b) it's already full (no point in using it),
104 // c) it's empty (this means that it was emptied during
105 // a cleanup and it should be on the free list now), or
106 // d) it's humongous (this means that it was emptied
107 // during a cleanup and was added to the free list, but
108 // has been subsequently used to allocate a humongous
109 // object that may be less than the region size).
110 if (retained_region != nullptr &&
111 !retained_region->in_collection_set() &&
112 !(retained_region->top() == retained_region->end()) &&
113 !retained_region->is_empty() &&
114 !retained_region->is_humongous()) {
115 // The retained region was added to the old region set when it was
116 // retired. We have to remove it now, since we don't allow regions
117 // we allocate to in the region sets. We'll re-add it later, when
118 // it's retired again.
119 _g1h->old_set_remove(retained_region);
120 old->reuse(retained_region);
121 G1HeapRegionPrinter::reuse(retained_region);
122 evacuation_info->set_alloc_regions_used_before(retained_region->used());
123 }
124 }
125
126 size_t G1Allocator::free_bytes_in_retained_old_region() const {
127 if (_retained_old_gc_alloc_region == nullptr) {
128 return 0;
129 } else {
130 return _retained_old_gc_alloc_region->free();
131 }
132 }
133
134 void G1Allocator::init_gc_alloc_regions(G1EvacInfo* evacuation_info) {
135 assert_at_safepoint_on_vm_thread();
136
137 _survivor_is_full = false;
138 _old_is_full = false;
139
140 for (uint i = 0; i < _num_alloc_regions; i++) {
141 survivor_gc_alloc_region(i)->init();
142 }
143
144 _old_gc_alloc_region.init();
145 reuse_retained_old_region(evacuation_info,
146 &_old_gc_alloc_region,
147 &_retained_old_gc_alloc_region);
148 }
149
150 void G1Allocator::release_gc_alloc_regions(G1EvacInfo* evacuation_info) {
151 uint survivor_region_count = 0;
152 for (uint node_index = 0; node_index < _num_alloc_regions; node_index++) {
153 survivor_region_count += survivor_gc_alloc_region(node_index)->count();
154 survivor_gc_alloc_region(node_index)->release();
155 }
156 evacuation_info->set_allocation_regions(survivor_region_count +
157 old_gc_alloc_region()->count());
158
159 // If we have an old GC alloc region to release, we'll save it in
160 // _retained_old_gc_alloc_region. If we don't
161 // _retained_old_gc_alloc_region will become null. This is what we
162 // want either way so no reason to check explicitly for either
163 // condition.
164 _retained_old_gc_alloc_region = old_gc_alloc_region()->release();
165 }
166
167 void G1Allocator::abandon_gc_alloc_regions() {
168 for (uint i = 0; i < _num_alloc_regions; i++) {
169 assert(survivor_gc_alloc_region(i)->get() == nullptr, "pre-condition");
170 }
171 assert(old_gc_alloc_region()->get() == nullptr, "pre-condition");
172 _retained_old_gc_alloc_region = nullptr;
173 }
174
175 bool G1Allocator::survivor_is_full() const {
176 return _survivor_is_full;
177 }
178
179 bool G1Allocator::old_is_full() const {
180 return _old_is_full;
181 }
182
183 void G1Allocator::set_survivor_full() {
184 _survivor_is_full = true;
185 }
186
187 void G1Allocator::set_old_full() {
188 _old_is_full = true;
189 }
190
191 size_t G1Allocator::unsafe_max_tlab_alloc() {
192 // Return the remaining space in the cur alloc region, but not less than
193 // the min TLAB size.
194
195 // Also, this value can be at most the humongous object threshold,
196 // since we can't allow tlabs to grow big enough to accommodate
197 // humongous objects.
198
199 uint node_index = current_node_index();
200 G1HeapRegion* hr = mutator_alloc_region(node_index)->get();
201 size_t max_tlab = _g1h->max_tlab_size() * wordSize;
202
203 if (hr == nullptr || hr->free() < MinTLABSize) {
204 // The next TLAB allocation will most probably happen in a new region,
205 // therefore we can attempt to allocate the maximum allowed TLAB size.
206 return max_tlab;
207 }
208
209 return MIN2(hr->free(), max_tlab);
210 }
211
212 size_t G1Allocator::used_in_alloc_regions() {
213 assert(Heap_lock->owner() != nullptr, "Should be owned on this thread's behalf.");
214 size_t used = 0;
215 for (uint i = 0; i < _num_alloc_regions; i++) {
216 used += mutator_alloc_region(i)->used_in_alloc_regions();
217 }
218 return used;
219 }
220
221
222 HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
223 uint node_index,
224 size_t word_size) {
225 size_t temp = 0;
226 HeapWord* result = par_allocate_during_gc(dest, node_index, word_size, word_size, &temp);
227 assert(result == nullptr || temp == word_size,
228 "Requested %zu words, but got %zu at " PTR_FORMAT,
229 word_size, temp, p2i(result));
230 return result;
231 }
232
233 HeapWord* G1Allocator::par_allocate_during_gc(G1HeapRegionAttr dest,
234 uint node_index,
235 size_t min_word_size,
236 size_t desired_word_size,
237 size_t* actual_word_size) {
238 switch (dest.type()) {
239 case G1HeapRegionAttr::Young:
240 return survivor_attempt_allocation(node_index, min_word_size, desired_word_size, actual_word_size);
241 case G1HeapRegionAttr::Old:
242 return old_attempt_allocation(min_word_size, desired_word_size, actual_word_size);
243 default:
244 ShouldNotReachHere();
245 return nullptr; // Keep some compilers happy
246 }
247 }
248
249 HeapWord* G1Allocator::survivor_attempt_allocation(uint node_index,
250 size_t min_word_size,
251 size_t desired_word_size,
252 size_t* actual_word_size) {
253 assert(!_g1h->is_humongous(desired_word_size),
254 "we should not be seeing humongous-size allocations in this path");
255
256 HeapWord* result = survivor_gc_alloc_region(node_index)->attempt_allocation(min_word_size,
257 desired_word_size,
258 actual_word_size);
259 if (result == nullptr && !survivor_is_full()) {
260 MutexLocker x(G1FreeList_lock, Mutex::_no_safepoint_check_flag);
261 // Multiple threads may have queued at the FreeList_lock above after checking whether there
262 // actually is still memory available. Redo the check under the lock to avoid unnecessary work;
263 // the memory may have been used up as the threads waited to acquire the lock.
264 if (!survivor_is_full()) {
265 result = survivor_gc_alloc_region(node_index)->attempt_allocation_locked(min_word_size,
266 desired_word_size,
267 actual_word_size);
268 if (result == nullptr) {
269 set_survivor_full();
270 }
271 }
272 }
273 return result;
274 }
275
276 HeapWord* G1Allocator::old_attempt_allocation(size_t min_word_size,
277 size_t desired_word_size,
278 size_t* actual_word_size) {
279 assert(!_g1h->is_humongous(desired_word_size),
280 "we should not be seeing humongous-size allocations in this path");
281
282 HeapWord* result = old_gc_alloc_region()->attempt_allocation(min_word_size,
283 desired_word_size,
284 actual_word_size);
285 if (result == nullptr && !old_is_full()) {
286 MutexLocker x(G1FreeList_lock, Mutex::_no_safepoint_check_flag);
287 // Multiple threads may have queued at the FreeList_lock above after checking whether there
288 // actually is still memory available. Redo the check under the lock to avoid unnecessary work;
289 // the memory may have been used up as the threads waited to acquire the lock.
290 if (!old_is_full()) {
291 result = old_gc_alloc_region()->attempt_allocation_locked(min_word_size,
292 desired_word_size,
293 actual_word_size);
294 if (result == nullptr) {
295 set_old_full();
296 }
297 }
298 }
299 return result;
300 }
301
302 G1PLABAllocator::PLABData::PLABData() :
303 _alloc_buffer(nullptr),
304 _direct_allocated(0),
305 _num_plab_fills(0),
306 _num_direct_allocations(0),
307 _plab_fill_counter(0),
308 _cur_desired_plab_size(0),
309 _num_alloc_buffers(0) { }
310
311 G1PLABAllocator::PLABData::~PLABData() {
312 if (_alloc_buffer == nullptr) {
313 return;
314 }
315 for (uint node_index = 0; node_index < _num_alloc_buffers; node_index++) {
316 delete _alloc_buffer[node_index];
317 }
318 FREE_C_HEAP_ARRAY(_alloc_buffer);
319 }
320
321 void G1PLABAllocator::PLABData::initialize(uint num_alloc_buffers, size_t desired_plab_size, size_t tolerated_refills) {
322 _num_alloc_buffers = num_alloc_buffers;
323 _alloc_buffer = NEW_C_HEAP_ARRAY(PLAB*, _num_alloc_buffers, mtGC);
324
325 for (uint node_index = 0; node_index < _num_alloc_buffers; node_index++) {
326 _alloc_buffer[node_index] = new PLAB(desired_plab_size);
327 }
328
329 _plab_fill_counter = tolerated_refills;
330 _cur_desired_plab_size = desired_plab_size;
331 }
332
333 void G1PLABAllocator::PLABData::notify_plab_refill(size_t tolerated_refills, size_t next_plab_size) {
334 _num_plab_fills++;
335 if (should_boost()) {
336 _plab_fill_counter = tolerated_refills;
337 _cur_desired_plab_size = next_plab_size;
338 } else {
339 _plab_fill_counter--;
340 }
341 }
342
343 G1PLABAllocator::G1PLABAllocator(G1Allocator* allocator) :
344 _g1h(G1CollectedHeap::heap()),
345 _allocator(allocator) {
346
347 if (ResizePLAB) {
348 // See G1EvacStats::compute_desired_plab_sz for the reasoning why this is the
349 // expected number of refills.
350 double const ExpectedNumberOfRefills = (100 - G1LastPLABAverageOccupancy) / TargetPLABWastePct;
351 // Add some padding to the threshold to not boost exactly when the targeted refills
352 // were reached.
353 // E.g. due to limitation of PLAB size to non-humongous objects and region boundaries
354 // a thread may experience more refills than expected. Keeping the PLAB waste low
355 // is the main goal, so being a bit conservative is better.
356 double const PadFactor = 1.5;
357 _tolerated_refills = MAX2(ExpectedNumberOfRefills, 1.0) * PadFactor;
358 } else {
359 // Make the tolerated refills a huge number.
360 _tolerated_refills = SIZE_MAX;
361 }
362 // The initial PLAB refill should not count, hence the +1 for the first boost.
363 size_t initial_tolerated_refills = ResizePLAB ? _tolerated_refills + 1 : _tolerated_refills;
364 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
365 _dest_data[state].initialize(alloc_buffers_length(state), _g1h->desired_plab_sz(state), initial_tolerated_refills);
366 }
367 }
368
369 bool G1PLABAllocator::may_throw_away_buffer(size_t const words_remaining, size_t const buffer_size) const {
370 return (words_remaining * 100 < buffer_size * ParallelGCBufferWastePct);
371 }
372
373 HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
374 size_t word_sz,
375 bool* plab_refill_failed,
376 uint node_index) {
377 PLAB* alloc_buf = alloc_buffer(dest, node_index);
378 size_t words_remaining = alloc_buf->words_remaining();
379 assert(words_remaining < word_sz, "precondition");
380
381 size_t plab_word_size = plab_size(dest.type());
382 size_t next_plab_word_size = plab_word_size;
383
384 PLABData* plab_data = &_dest_data[dest.type()];
385
386 if (plab_data->should_boost()) {
387 next_plab_word_size = _g1h->clamp_plab_size(next_plab_word_size * 2);
388 }
389
390 size_t required_in_plab = PLAB::size_required_for_allocation(word_sz);
391
392 // Only get a new PLAB if the allocation fits into the to-be-allocated PLAB and
393 // retiring the current PLAB would not waste more than ParallelGCBufferWastePct
394 // in the current PLAB. Boosting the PLAB also increasingly allows more waste to occur.
395 if ((required_in_plab <= next_plab_word_size) &&
396 may_throw_away_buffer(words_remaining, plab_word_size)) {
397
398 alloc_buf->retire();
399
400 plab_data->notify_plab_refill(_tolerated_refills, next_plab_word_size);
401 plab_word_size = next_plab_word_size;
402
403 size_t actual_plab_size = 0;
404 HeapWord* buf = _allocator->par_allocate_during_gc(dest,
405 node_index,
406 required_in_plab,
407 plab_word_size,
408 &actual_plab_size);
409
410 assert(buf == nullptr || ((actual_plab_size >= required_in_plab) && (actual_plab_size <= plab_word_size)),
411 "Requested at minimum %zu, desired %zu words, but got %zu at " PTR_FORMAT,
412 required_in_plab, plab_word_size, actual_plab_size, p2i(buf));
413
414 if (buf != nullptr) {
415 alloc_buf->set_buf(buf, actual_plab_size);
416
417 HeapWord* const obj = alloc_buf->allocate(word_sz);
418 assert(obj != nullptr, "PLAB should have been big enough, tried to allocate "
419 "%zu requiring %zu PLAB size %zu",
420 word_sz, required_in_plab, plab_word_size);
421 return obj;
422 }
423 // Otherwise.
424 *plab_refill_failed = true;
425 }
426 // Try direct allocation.
427 HeapWord* result = _allocator->par_allocate_during_gc(dest, node_index, word_sz);
428 if (result != nullptr) {
429 plab_data->_direct_allocated += word_sz;
430 plab_data->_num_direct_allocations++;
431 }
432 return result;
433 }
434
435 void G1PLABAllocator::undo_allocation(G1HeapRegionAttr dest, HeapWord* obj, size_t word_sz, uint node_index) {
436 alloc_buffer(dest, node_index)->undo_allocation(obj, word_sz);
437 }
438
439 void G1PLABAllocator::flush_and_retire_stats(uint num_workers) {
440 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
441 G1EvacStats* stats = _g1h->alloc_buffer_stats(state);
442 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
443 PLAB* const buf = alloc_buffer(state, node_index);
444 if (buf != nullptr) {
445 buf->flush_and_retire_stats(stats);
446 }
447 }
448 PLABData* plab_data = &_dest_data[state];
449 stats->add_num_plab_filled(plab_data->_num_plab_fills);
450 stats->add_direct_allocated(plab_data->_direct_allocated);
451 stats->add_num_direct_allocated(plab_data->_num_direct_allocations);
452 }
453
454 log_trace(gc, plab)("PLAB boost: Young %zu -> %zu refills %zu (tolerated %zu) Old %zu -> %zu refills %zu (tolerated %zu)",
455 _g1h->alloc_buffer_stats(G1HeapRegionAttr::Young)->desired_plab_size(num_workers),
456 plab_size(G1HeapRegionAttr::Young),
457 _dest_data[G1HeapRegionAttr::Young]._num_plab_fills,
458 _tolerated_refills,
459 _g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->desired_plab_size(num_workers),
460 plab_size(G1HeapRegionAttr::Old),
461 _dest_data[G1HeapRegionAttr::Old]._num_plab_fills,
462 _tolerated_refills);
463 }
464
465 size_t G1PLABAllocator::waste() const {
466 size_t result = 0;
467 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
468 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
469 PLAB* const buf = alloc_buffer(state, node_index);
470 if (buf != nullptr) {
471 result += buf->waste();
472 }
473 }
474 }
475 return result;
476 }
477
478 size_t G1PLABAllocator::plab_size(G1HeapRegionAttr which) const {
479 return _dest_data[which.type()]._cur_desired_plab_size;
480 }
481
482 size_t G1PLABAllocator::undo_waste() const {
483 size_t result = 0;
484 for (region_type_t state = 0; state < G1HeapRegionAttr::Num; state++) {
485 for (uint node_index = 0; node_index < alloc_buffers_length(state); node_index++) {
486 PLAB* const buf = alloc_buffer(state, node_index);
487 if (buf != nullptr) {
488 result += buf->undo_waste();
489 }
490 }
491 }
492 return result;
493 }