1 /* 2 * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2020, 2023 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "logging/log.hpp" 28 #include "logging/logStream.hpp" 29 #include "memory/metaspace/chunkManager.hpp" 30 #include "memory/metaspace/counters.hpp" 31 #include "memory/metaspace/freeBlocks.hpp" 32 #include "memory/metaspace/internalStats.hpp" 33 #include "memory/metaspace/metablock.inline.hpp" 34 #include "memory/metaspace/metachunk.hpp" 35 #include "memory/metaspace/metaspaceArena.hpp" 36 #include "memory/metaspace/metaspaceArenaGrowthPolicy.hpp" 37 #include "memory/metaspace/metaspaceCommon.hpp" 38 #include "memory/metaspace/metaspaceContext.hpp" 39 #include "memory/metaspace/metaspaceSettings.hpp" 40 #include "memory/metaspace/metaspaceStatistics.hpp" 41 #include "memory/metaspace/virtualSpaceList.hpp" 42 #include "runtime/atomic.hpp" 43 #include "runtime/init.hpp" 44 #include "runtime/mutexLocker.hpp" 45 #include "services/memoryService.hpp" 46 #include "utilities/align.hpp" 47 #include "utilities/debug.hpp" 48 #include "utilities/globalDefinitions.hpp" 49 50 namespace metaspace { 51 52 #define LOGFMT "Arena @" PTR_FORMAT " (%s)" 53 #define LOGFMT_ARGS p2i(this), this->_name 54 55 // Returns the level of the next chunk to be added, acc to growth policy. 56 chunklevel_t MetaspaceArena::next_chunk_level() const { 57 const int growth_step = _chunks.count(); 58 return _growth_policy->get_level_at_step(growth_step); 59 } 60 61 // Given a chunk, return the committed remainder of this chunk. 62 MetaBlock MetaspaceArena::salvage_chunk(Metachunk* c) { 63 MetaBlock result; 64 const size_t remaining_words = c->free_below_committed_words(); 65 if (remaining_words >= FreeBlocks::MinWordSize) { 66 67 UL2(trace, "salvaging chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); 68 69 MetaWord* ptr = c->allocate(remaining_words); 70 assert(ptr != nullptr, "Should have worked"); 71 72 result = MetaBlock(ptr, remaining_words); 73 74 // After this operation: the chunk should have no free committed space left. 75 assert(c->free_below_committed_words() == 0, 76 "Salvaging chunk failed (chunk " METACHUNK_FULL_FORMAT ").", 77 METACHUNK_FULL_FORMAT_ARGS(c)); 78 } 79 return result; 80 } 81 82 // Allocate a new chunk from the underlying chunk manager able to hold at least 83 // requested word size. 84 Metachunk* MetaspaceArena::allocate_new_chunk(size_t requested_word_size) { 85 // Should this ever happen, we need to increase the maximum possible chunk size. 86 guarantee(requested_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE, 87 "Requested size too large (" SIZE_FORMAT ") - max allowed size per allocation is " SIZE_FORMAT ".", 88 requested_word_size, chunklevel::MAX_CHUNK_WORD_SIZE); 89 90 const chunklevel_t max_level = chunklevel::level_fitting_word_size(requested_word_size); 91 const chunklevel_t preferred_level = MIN2(max_level, next_chunk_level()); 92 93 Metachunk* c = _chunk_manager->get_chunk(preferred_level, max_level, requested_word_size); 94 if (c == nullptr) { 95 return nullptr; 96 } 97 98 assert(c->is_in_use(), "Wrong chunk state."); 99 assert(c->free_below_committed_words() >= requested_word_size, "Chunk not committed"); 100 return c; 101 } 102 103 void MetaspaceArena::add_allocation_to_fbl(MetaBlock bl) { 104 assert(bl.is_nonempty(), "Sanity"); 105 assert_block_base_aligned(bl, allocation_alignment_words()); 106 assert_block_size_aligned(bl, Metaspace::min_allocation_alignment_words); 107 if (_fbl == nullptr) { 108 _fbl = new FreeBlocks(); // Create only on demand 109 } 110 _fbl->add_block(bl); 111 } 112 113 MetaspaceArena::MetaspaceArena(MetaspaceContext* context, 114 const ArenaGrowthPolicy* growth_policy, 115 size_t allocation_alignment_words, 116 const char* name) : 117 _allocation_alignment_words(allocation_alignment_words), 118 _chunk_manager(context->cm()), 119 _growth_policy(growth_policy), 120 _chunks(), 121 _fbl(nullptr), 122 _total_used_words_counter(context->used_words_counter()), 123 _name(name) 124 #ifdef ASSERT 125 , _first_fence(nullptr) 126 #endif 127 { 128 // Check arena allocation alignment 129 assert(is_power_of_2(_allocation_alignment_words) && 130 _allocation_alignment_words >= Metaspace::min_allocation_alignment_words && 131 _allocation_alignment_words <= chunklevel::MIN_CHUNK_WORD_SIZE, 132 "Invalid alignment: %zu", _allocation_alignment_words); 133 134 UL(debug, "born."); 135 136 // Update statistics 137 InternalStats::inc_num_arena_births(); 138 } 139 140 MetaspaceArena::~MetaspaceArena() { 141 #ifdef ASSERT 142 SOMETIMES(verify();) 143 if (Settings::use_allocation_guard()) { 144 verify_allocation_guards(); 145 } 146 #endif 147 MemRangeCounter return_counter; 148 149 Metachunk* c = _chunks.first(); 150 Metachunk* c2 = nullptr; 151 152 while (c) { 153 c2 = c->next(); 154 return_counter.add(c->used_words()); 155 DEBUG_ONLY(c->set_prev(nullptr);) 156 DEBUG_ONLY(c->set_next(nullptr);) 157 UL2(debug, "return chunk: " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); 158 _chunk_manager->return_chunk(c); 159 // c may be invalid after return_chunk(c) was called. Don't access anymore. 160 c = c2; 161 } 162 163 UL2(debug, "returned %d chunks, total capacity " SIZE_FORMAT " words.", 164 return_counter.count(), return_counter.total_size()); 165 166 _total_used_words_counter->decrement_by(return_counter.total_size()); 167 SOMETIMES(chunk_manager()->verify();) 168 delete _fbl; 169 UL(debug, ": dies."); 170 171 // Update statistics 172 InternalStats::inc_num_arena_deaths(); 173 } 174 175 // Attempt to enlarge the current chunk to make it large enough to hold at least 176 // requested_word_size additional words. 177 // 178 // On success, true is returned, false otherwise. 179 bool MetaspaceArena::attempt_enlarge_current_chunk(size_t requested_word_size) { 180 Metachunk* c = current_chunk(); 181 assert(c->free_words() < requested_word_size, "Sanity"); 182 183 // Not if chunk enlargement is switched off... 184 if (Settings::enlarge_chunks_in_place() == false) { 185 return false; 186 } 187 // ... nor if we are already a root chunk ... 188 if (c->is_root_chunk()) { 189 return false; 190 } 191 // ... nor if the combined size of chunk content and new content would bring us above the size of a root chunk ... 192 if ((c->used_words() + requested_word_size) > metaspace::chunklevel::MAX_CHUNK_WORD_SIZE) { 193 return false; 194 } 195 196 const chunklevel_t new_level = 197 chunklevel::level_fitting_word_size(c->used_words() + requested_word_size); 198 assert(new_level < c->level(), "Sanity"); 199 200 // Atm we only enlarge by one level (so, doubling the chunk in size). So, if the requested enlargement 201 // would require the chunk to more than double in size, we bail. But this covers about 99% of all cases, 202 // so this is good enough. 203 if (new_level < c->level() - 1) { 204 return false; 205 } 206 // This only works if chunk is the leader of its buddy pair (and also if buddy 207 // is free and unsplit, but that we cannot check outside of metaspace lock). 208 if (!c->is_leader()) { 209 return false; 210 } 211 // If the size added to the chunk would be larger than allowed for the next growth step 212 // dont enlarge. 213 if (next_chunk_level() > c->level()) { 214 return false; 215 } 216 217 bool success = _chunk_manager->attempt_enlarge_chunk(c); 218 assert(success == false || c->free_words() >= requested_word_size, "Sanity"); 219 return success; 220 } 221 222 // Allocate memory from Metaspace. 223 // 1) Attempt to allocate from the free block list. 224 // 2) Attempt to allocate from the current chunk. 225 // 3) Attempt to enlarge the current chunk in place if it is too small. 226 // 4) Attempt to get a new chunk and allocate from that chunk. 227 // At any point, if we hit a commit limit, we return null. 228 MetaBlock MetaspaceArena::allocate(size_t requested_word_size, MetaBlock& wastage) { 229 UL2(trace, "requested " SIZE_FORMAT " words.", requested_word_size); 230 231 const size_t aligned_word_size = get_raw_word_size_for_requested_word_size(requested_word_size); 232 233 MetaBlock result; 234 bool taken_from_fbl = false; 235 236 // Before bothering the arena proper, attempt to re-use a block from the free blocks list 237 if (_fbl != nullptr && !_fbl->is_empty()) { 238 result = _fbl->remove_block(aligned_word_size); 239 if (result.is_nonempty()) { 240 assert_block_larger_or_equal(result, aligned_word_size); 241 assert_block_base_aligned(result, allocation_alignment_words()); 242 assert_block_size_aligned(result, Metaspace::min_allocation_alignment_words); 243 // Split off wastage 244 wastage = result.split_off_tail(result.word_size() - aligned_word_size); 245 // Stats, logging 246 DEBUG_ONLY(InternalStats::inc_num_allocs_from_deallocated_blocks();) 247 UL2(trace, "returning " METABLOCKFORMAT " with wastage " METABLOCKFORMAT " - taken from fbl (now: %d, " SIZE_FORMAT ").", 248 METABLOCKFORMATARGS(result), METABLOCKFORMATARGS(wastage), _fbl->count(), _fbl->total_size()); 249 // Note: free blocks in freeblock dictionary still count as "used" as far as statistics go; 250 // therefore we don't need to adjust any usage counters (see epilogue of allocate_inner()). 251 taken_from_fbl = true; 252 } 253 } 254 255 if (result.is_empty()) { 256 // Free-block allocation failed; we allocate from the arena. 257 // These allocations are fenced. 258 size_t plus_fence = 0; 259 #ifdef ASSERT 260 static constexpr size_t fence_word_size = sizeof(Fence) / BytesPerWord; 261 STATIC_ASSERT(is_aligned(fence_word_size, Metaspace::min_allocation_alignment_words)); 262 if (Settings::use_allocation_guard() && 263 aligned_word_size <= Metaspace::max_allocation_word_size() - fence_word_size) { 264 plus_fence = fence_word_size; 265 } 266 #endif 267 268 // Allocate from arena proper 269 result = allocate_inner(aligned_word_size + plus_fence, wastage); 270 271 #ifdef ASSERT 272 if (result.is_nonempty() && plus_fence > 0) { 273 assert(result.word_size() == aligned_word_size + plus_fence, "Sanity"); 274 MetaBlock fenceblock = result.split_off_tail(fence_word_size); 275 Fence* f = new(fenceblock.base()) Fence(_first_fence); 276 _first_fence = f; 277 } 278 #endif 279 } // End: allocate from arena proper 280 281 // Logging 282 if (result.is_nonempty()) { 283 LogTarget(Trace, metaspace) lt; 284 if (lt.is_enabled()) { 285 LogStream ls(lt); 286 ls.print(LOGFMT ": returning " METABLOCKFORMAT " taken from %s, ", LOGFMT_ARGS, 287 METABLOCKFORMATARGS(result), (taken_from_fbl ? "fbl" : "arena")); 288 if (wastage.is_empty()) { 289 ls.print("no wastage"); 290 } else { 291 ls.print("wastage " METABLOCKFORMAT, METABLOCKFORMATARGS(wastage)); 292 } 293 } 294 } else { 295 UL(info, "allocation failed, returned null."); 296 } 297 298 // Final sanity checks 299 #ifdef ASSERT 300 result.verify(); 301 wastage.verify(); 302 if (result.is_nonempty()) { 303 assert(result.word_size() == aligned_word_size && 304 is_aligned(result.base(), _allocation_alignment_words * BytesPerWord), 305 "result bad or unaligned: " METABLOCKFORMAT ".", METABLOCKFORMATARGS(result)); 306 } 307 if (wastage.is_nonempty()) { 308 assert(wastage.is_empty() || 309 (wastage.is_aligned_base(Metaspace::min_allocation_alignment_words) && 310 wastage.is_aligned_size(Metaspace::min_allocation_alignment_words)), 311 "Misaligned wastage: " METABLOCKFORMAT".", METABLOCKFORMATARGS(wastage)); 312 } 313 #endif // ASSERT 314 315 return result; 316 } 317 318 // Allocate from the arena proper, once dictionary allocations and fencing are sorted out. 319 MetaBlock MetaspaceArena::allocate_inner(size_t word_size, MetaBlock& wastage) { 320 321 MetaBlock result; 322 bool current_chunk_too_small = false; 323 bool commit_failure = false; 324 size_t alignment_gap_size = 0; 325 326 if (current_chunk() != nullptr) { 327 // Attempt to satisfy the allocation from the current chunk. 328 329 const MetaWord* const chunk_top = current_chunk()->top(); 330 alignment_gap_size = align_up(chunk_top, _allocation_alignment_words * BytesPerWord) - chunk_top; 331 const size_t word_size_plus_alignment = word_size + alignment_gap_size; 332 333 // If the current chunk is too small to hold the requested size, attempt to enlarge it. 334 // If that fails, retire the chunk. 335 if (current_chunk()->free_words() < word_size_plus_alignment) { 336 if (!attempt_enlarge_current_chunk(word_size_plus_alignment)) { 337 current_chunk_too_small = true; 338 } else { 339 DEBUG_ONLY(InternalStats::inc_num_chunks_enlarged();) 340 UL(debug, "enlarged chunk."); 341 } 342 } 343 344 // Commit the chunk far enough to hold the requested word size. If that fails, we 345 // hit a limit (either GC threshold or MaxMetaspaceSize). In that case retire the 346 // chunk. 347 if (!current_chunk_too_small) { 348 if (!current_chunk()->ensure_committed_additional(word_size_plus_alignment)) { 349 UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", word_size_plus_alignment); 350 commit_failure = true; 351 } 352 } 353 354 // Allocate from the current chunk. This should work now. 355 if (!current_chunk_too_small && !commit_failure) { 356 MetaWord* const p_gap = current_chunk()->allocate(word_size_plus_alignment); 357 assert(p_gap != nullptr, "Allocation from chunk failed."); 358 MetaWord* const p_user_allocation = p_gap + alignment_gap_size; 359 result = MetaBlock(p_user_allocation, word_size); 360 if (alignment_gap_size > 0) { 361 NOT_LP64(assert(alignment_gap_size >= AllocationAlignmentWordSize, "Sanity")); 362 wastage = MetaBlock(p_gap, alignment_gap_size); 363 } 364 } 365 } 366 367 if (result.is_empty()) { 368 // If we are here, we either had no current chunk to begin with or it was deemed insufficient. 369 assert(current_chunk() == nullptr || 370 current_chunk_too_small || commit_failure, "Sanity"); 371 372 Metachunk* new_chunk = allocate_new_chunk(word_size); 373 if (new_chunk != nullptr) { 374 UL2(debug, "allocated new chunk " METACHUNK_FORMAT " for requested word size " SIZE_FORMAT ".", 375 METACHUNK_FORMAT_ARGS(new_chunk), word_size); 376 377 assert(new_chunk->free_below_committed_words() >= word_size, "Sanity"); 378 379 // We have a new chunk. Before making it the current chunk, retire the old one. 380 if (current_chunk() != nullptr) { 381 wastage = salvage_chunk(current_chunk()); 382 DEBUG_ONLY(InternalStats::inc_num_chunks_retired();) 383 } 384 385 _chunks.add(new_chunk); 386 387 // Now, allocate from that chunk. That should work. Note that the resulting allocation 388 // is guaranteed to be aligned to arena alignment, since arena alignment cannot be larger 389 // than smallest chunk size, and chunk starts are aligned by their size (buddy allocation). 390 MetaWord* const p = current_chunk()->allocate(word_size); 391 assert(p != nullptr, "Allocation from chunk failed."); 392 result = MetaBlock(p, word_size); 393 } else { 394 UL2(info, "failed to allocate new chunk for requested word size " SIZE_FORMAT ".", word_size); 395 } 396 } 397 398 if (result.is_empty()) { 399 InternalStats::inc_num_allocs_failed_limit(); 400 } else { 401 DEBUG_ONLY(InternalStats::inc_num_allocs();) 402 _total_used_words_counter->increment_by(word_size + wastage.word_size()); 403 } 404 405 SOMETIMES(verify();) 406 407 if (result.is_nonempty()) { 408 UL2(trace, "after allocation: %u chunk(s), current:" METACHUNK_FULL_FORMAT, 409 _chunks.count(), METACHUNK_FULL_FORMAT_ARGS(current_chunk())); 410 } 411 412 #ifdef ASSERT 413 if (wastage.is_nonempty()) { 414 // Wastage from arena allocations only occurs if either or both are true: 415 // - it is too small to hold the requested allocation words 416 // - it is misaligned 417 assert(!wastage.is_aligned_base(allocation_alignment_words()) || 418 wastage.word_size() < word_size, 419 "Unexpected wastage: " METABLOCKFORMAT ", arena alignment: %zu, allocation word size: %zu", 420 METABLOCKFORMATARGS(wastage), allocation_alignment_words(), word_size); 421 wastage.verify(); 422 } 423 #endif // ASSERT 424 425 return result; 426 } 427 428 // Prematurely returns a metaspace allocation to the _block_freelists 429 // because it is not needed anymore (requires CLD lock to be active). 430 void MetaspaceArena::deallocate(MetaBlock block) { 431 DEBUG_ONLY(block.verify();) 432 // This only matters on 32-bit: 433 // Since we always align up allocations from arena, we align up here, too. 434 #ifndef _LP64 435 MetaBlock raw_block(block.base(), get_raw_word_size_for_requested_word_size(block.word_size())); 436 add_allocation_to_fbl(raw_block); 437 #else 438 add_allocation_to_fbl(block); 439 #endif 440 UL2(trace, "added to fbl: " METABLOCKFORMAT ", (now: %d, " SIZE_FORMAT ").", 441 METABLOCKFORMATARGS(block), _fbl->count(), _fbl->total_size()); 442 SOMETIMES(verify();) 443 } 444 445 // Update statistics. This walks all in-use chunks. 446 void MetaspaceArena::add_to_statistics(ArenaStats* out) const { 447 for (const Metachunk* c = _chunks.first(); c != nullptr; c = c->next()) { 448 InUseChunkStats& ucs = out->_stats[c->level()]; 449 ucs._num++; 450 ucs._word_size += c->word_size(); 451 ucs._committed_words += c->committed_words(); 452 ucs._used_words += c->used_words(); 453 // Note: for free and waste, we only count what's committed. 454 if (c == current_chunk()) { 455 ucs._free_words += c->free_below_committed_words(); 456 } else { 457 ucs._waste_words += c->free_below_committed_words(); 458 } 459 } 460 461 if (_fbl != nullptr) { 462 out->_free_blocks_num += _fbl->count(); 463 out->_free_blocks_word_size += _fbl->total_size(); 464 } 465 466 SOMETIMES(out->verify();) 467 } 468 469 // Convenience method to get the most important usage statistics. 470 // For deeper analysis use add_to_statistics(). 471 void MetaspaceArena::usage_numbers(size_t* p_used_words, size_t* p_committed_words, size_t* p_capacity_words) const { 472 size_t used = 0, comm = 0, cap = 0; 473 for (const Metachunk* c = _chunks.first(); c != nullptr; c = c->next()) { 474 used += c->used_words(); 475 comm += c->committed_words(); 476 cap += c->word_size(); 477 } 478 if (p_used_words != nullptr) { 479 *p_used_words = used; 480 } 481 if (p_committed_words != nullptr) { 482 *p_committed_words = comm; 483 } 484 if (p_capacity_words != nullptr) { 485 *p_capacity_words = cap; 486 } 487 } 488 489 #ifdef ASSERT 490 491 void MetaspaceArena::verify() const { 492 assert(_growth_policy != nullptr && _chunk_manager != nullptr, "Sanity"); 493 _chunks.verify(); 494 if (_fbl != nullptr) { 495 _fbl->verify(); 496 } 497 } 498 499 void MetaspaceArena::Fence::verify() const { 500 assert(_eye1 == EyeCatcher && _eye2 == EyeCatcher, 501 "Metaspace corruption: fence block at " PTR_FORMAT " broken.", p2i(this)); 502 } 503 504 void MetaspaceArena::verify_allocation_guards() const { 505 assert(Settings::use_allocation_guard(), "Don't call with guards disabled."); 506 for (const Fence* f = _first_fence; f != nullptr; f = f->next()) { 507 f->verify(); 508 } 509 } 510 511 // Returns true if the given block is contained in this arena 512 // Returns true if the given block is contained in this arena 513 bool MetaspaceArena::contains(MetaBlock bl) const { 514 DEBUG_ONLY(bl.verify();) 515 assert(bl.is_nonempty(), "Sanity"); 516 bool found = false; 517 for (const Metachunk* c = _chunks.first(); c != nullptr && !found; c = c->next()) { 518 assert(c->is_valid_committed_pointer(bl.base()) == 519 c->is_valid_committed_pointer(bl.end() - 1), "range intersects"); 520 found = c->is_valid_committed_pointer(bl.base()); 521 } 522 return found; 523 } 524 525 #endif // ASSERT 526 527 void MetaspaceArena::print_on(outputStream* st) const { 528 st->print_cr("sm %s: %d chunks, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT, _name, 529 _chunks.count(), _chunks.calc_word_size(), _chunks.calc_committed_word_size()); 530 _chunks.print_on(st); 531 st->cr(); 532 st->print_cr("growth-policy " PTR_FORMAT ", cm " PTR_FORMAT ", fbl " PTR_FORMAT, 533 p2i(_growth_policy), p2i(_chunk_manager), p2i(_fbl)); 534 } 535 536 } // namespace metaspace 537 --- EOF ---