1 /* 2 * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2020 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "logging/log.hpp" 28 #include "logging/logStream.hpp" 29 #include "memory/metaspace/chunkManager.hpp" 30 #include "memory/metaspace/counters.hpp" 31 #include "memory/metaspace/freeBlocks.hpp" 32 #include "memory/metaspace/internalStats.hpp" 33 #include "memory/metaspace/metachunk.hpp" 34 #include "memory/metaspace/metaspaceAlignment.hpp" 35 #include "memory/metaspace/metaspaceArena.hpp" 36 #include "memory/metaspace/metaspaceArenaGrowthPolicy.hpp" 37 #include "memory/metaspace/metaspaceCommon.hpp" 38 #include "memory/metaspace/metaspaceSettings.hpp" 39 #include "memory/metaspace/metaspaceStatistics.hpp" 40 #include "memory/metaspace/virtualSpaceList.hpp" 41 #include "runtime/atomic.hpp" 42 #include "runtime/init.hpp" 43 #include "runtime/mutexLocker.hpp" 44 #include "services/memoryService.hpp" 45 #include "utilities/align.hpp" 46 #include "utilities/debug.hpp" 47 #include "utilities/globalDefinitions.hpp" 48 49 namespace metaspace { 50 51 #define LOGFMT "Arena @" PTR_FORMAT " (%s)" 52 #define LOGFMT_ARGS p2i(this), this->_name 53 54 // Returns the level of the next chunk to be added, acc to growth policy. 55 chunklevel_t MetaspaceArena::next_chunk_level() const { 56 const int growth_step = _chunks.count(); 57 return _growth_policy->get_level_at_step(growth_step); 58 } 59 60 // Given a chunk, add its remaining free committed space to the free block list. 61 void MetaspaceArena::salvage_chunk(Metachunk* c) { 62 assert_lock_strong(lock()); 63 size_t remaining_words = c->free_below_committed_words(); 64 if (remaining_words > FreeBlocks::MinWordSize) { 65 66 UL2(trace, "salvaging chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); 67 68 MetaWord* ptr = c->allocate(remaining_words); 69 assert(ptr != NULL, "Should have worked"); 70 _total_used_words_counter->increment_by(remaining_words); 71 72 add_allocation_to_fbl(ptr, remaining_words); 73 74 // After this operation: the chunk should have no free committed space left. 75 assert(c->free_below_committed_words() == 0, 76 "Salvaging chunk failed (chunk " METACHUNK_FULL_FORMAT ").", 77 METACHUNK_FULL_FORMAT_ARGS(c)); 78 } 79 } 80 81 // Allocate a new chunk from the underlying chunk manager able to hold at least 82 // requested word size. 83 Metachunk* MetaspaceArena::allocate_new_chunk(size_t requested_word_size) { 84 assert_lock_strong(lock()); 85 86 // Should this ever happen, we need to increase the maximum possible chunk size. 87 guarantee(requested_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE, 88 "Requested size too large (" SIZE_FORMAT ") - max allowed size per allocation is " SIZE_FORMAT ".", 89 requested_word_size, chunklevel::MAX_CHUNK_WORD_SIZE); 90 91 const chunklevel_t max_level = chunklevel::level_fitting_word_size(requested_word_size); 92 const chunklevel_t preferred_level = MIN2(max_level, next_chunk_level()); 93 94 Metachunk* c = _chunk_manager->get_chunk(preferred_level, max_level, requested_word_size); 95 if (c == NULL) { 96 return NULL; 97 } 98 99 assert(c->is_in_use(), "Wrong chunk state."); 100 assert(c->free_below_committed_words() >= requested_word_size, "Chunk not committed"); 101 return c; 102 } 103 104 void MetaspaceArena::add_allocation_to_fbl(MetaWord* p, size_t word_size) { 105 if (_fbl == NULL) { 106 _fbl = new FreeBlocks(); // Create only on demand 107 } 108 _fbl->add_block(p, word_size); 109 } 110 111 MetaspaceArena::MetaspaceArena(ChunkManager* chunk_manager, const ArenaGrowthPolicy* growth_policy, int alignment_words, 112 Mutex* lock, SizeAtomicCounter* total_used_words_counter, 113 const char* name) : 114 _lock(lock), 115 _chunk_manager(chunk_manager), 116 _growth_policy(growth_policy), 117 _chunks(), 118 _alignment_words(alignment_words), 119 _fbl(NULL), 120 _total_used_words_counter(total_used_words_counter), 121 _name(name) 122 #ifdef ASSERT 123 , _first_fence(NULL) 124 #endif 125 126 { 127 UL(debug, ": born."); 128 129 // Update statistics 130 InternalStats::inc_num_arena_births(); 131 } 132 133 MetaspaceArena::~MetaspaceArena() { 134 #ifdef ASSERT 135 verify(); 136 if (Settings::use_allocation_guard()) { 137 verify_allocation_guards(); 138 } 139 #endif 140 141 MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag); 142 MemRangeCounter return_counter; 143 144 Metachunk* c = _chunks.first(); 145 Metachunk* c2 = NULL; 146 147 while (c) { 148 c2 = c->next(); 149 return_counter.add(c->used_words()); 150 DEBUG_ONLY(c->set_prev(NULL);) 151 DEBUG_ONLY(c->set_next(NULL);) 152 UL2(debug, "return chunk: " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); 153 _chunk_manager->return_chunk(c); 154 // c may be invalid after return_chunk(c) was called. Don't access anymore. 155 c = c2; 156 } 157 158 UL2(info, "returned %d chunks, total capacity " SIZE_FORMAT " words.", 159 return_counter.count(), return_counter.total_size()); 160 161 _total_used_words_counter->decrement_by(return_counter.total_size()); 162 DEBUG_ONLY(chunk_manager()->verify();) 163 delete _fbl; 164 UL(debug, ": dies."); 165 166 // Update statistics 167 InternalStats::inc_num_arena_deaths(); 168 } 169 170 // Attempt to enlarge the current chunk to make it large enough to hold at least 171 // requested_word_size additional words. 172 // 173 // On success, true is returned, false otherwise. 174 bool MetaspaceArena::attempt_enlarge_current_chunk(size_t requested_word_size) { 175 assert_lock_strong(lock()); 176 177 Metachunk* c = current_chunk(); 178 assert(c->free_words() < requested_word_size, "Sanity"); 179 180 // Not if chunk enlargement is switched off... 181 if (Settings::enlarge_chunks_in_place() == false) { 182 return false; 183 } 184 // ... nor if we are already a root chunk ... 185 if (c->is_root_chunk()) { 186 return false; 187 } 188 // ... nor if the combined size of chunk content and new content would bring us above the size of a root chunk ... 189 if ((c->used_words() + requested_word_size) > metaspace::chunklevel::MAX_CHUNK_WORD_SIZE) { 190 return false; 191 } 192 193 const chunklevel_t new_level = 194 chunklevel::level_fitting_word_size(c->used_words() + requested_word_size); 195 assert(new_level < c->level(), "Sanity"); 196 197 // Atm we only enlarge by one level (so, doubling the chunk in size). So, if the requested enlargement 198 // would require the chunk to more than double in size, we bail. But this covers about 99% of all cases, 199 // so this is good enough. 200 if (new_level < c->level() - 1) { 201 return false; 202 } 203 // This only works if chunk is the leader of its buddy pair (and also if buddy 204 // is free and unsplit, but that we cannot check outside of metaspace lock). 205 if (!c->is_leader()) { 206 return false; 207 } 208 // If the size added to the chunk would be larger than allowed for the next growth step 209 // dont enlarge. 210 if (next_chunk_level() > c->level()) { 211 return false; 212 } 213 214 bool success = _chunk_manager->attempt_enlarge_chunk(c); 215 assert(success == false || c->free_words() >= requested_word_size, "Sanity"); 216 return success; 217 } 218 219 // Allocate memory from Metaspace. 220 // 1) Attempt to allocate from the free block list. 221 // 2) Attempt to allocate from the current chunk. 222 // 3) Attempt to enlarge the current chunk in place if it is too small. 223 // 4) Attempt to get a new chunk and allocate from that chunk. 224 // At any point, if we hit a commit limit, we return NULL. 225 MetaWord* MetaspaceArena::allocate(size_t requested_word_size) { 226 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); 227 UL2(trace, "requested " SIZE_FORMAT " words.", requested_word_size); 228 229 MetaWord* p = NULL; 230 const size_t raw_word_size = get_raw_word_size_for_requested_word_size(requested_word_size, _alignment_words); 231 232 // Before bothering the arena proper, attempt to re-use a block from the free blocks list 233 if (_fbl != NULL && !_fbl->is_empty()) { 234 p = _fbl->remove_block(raw_word_size); 235 if (p != NULL) { 236 DEBUG_ONLY(InternalStats::inc_num_allocs_from_deallocated_blocks();) 237 UL2(trace, "taken from fbl (now: %d, " SIZE_FORMAT ").", 238 _fbl->count(), _fbl->total_size()); 239 // Note: free blocks in freeblock dictionary still count as "used" as far as statistics go; 240 // therefore we have no need to adjust any usage counters (see epilogue of allocate_inner()) 241 // and can just return here. 242 return p; 243 } 244 } 245 246 // Primary allocation 247 p = allocate_inner(requested_word_size); 248 249 #ifdef ASSERT 250 // Fence allocation 251 if (p != NULL && Settings::use_allocation_guard()) { 252 STATIC_ASSERT(is_aligned(sizeof(Fence), BytesPerWord)); 253 MetaWord* guard = allocate_inner(sizeof(Fence) / BytesPerWord); 254 if (guard != NULL) { 255 // Ignore allocation errors for the fence to keep coding simple. If this 256 // happens (e.g. because right at this time we hit the Metaspace GC threshold) 257 // we miss adding this one fence. Not a big deal. Note that his would 258 // be pretty rare. Chances are much higher the primary allocation above 259 // would have already failed). 260 Fence* f = new(guard) Fence(_first_fence); 261 _first_fence = f; 262 } 263 } 264 #endif // ASSERT 265 266 return p; 267 } 268 269 // Allocate from the arena proper, once dictionary allocations and fencing are sorted out. 270 MetaWord* MetaspaceArena::allocate_inner(size_t requested_word_size) { 271 272 assert_lock_strong(lock()); 273 274 const size_t raw_word_size = get_raw_word_size_for_requested_word_size(requested_word_size, _alignment_words); 275 MetaWord* p = NULL; 276 bool current_chunk_too_small = false; 277 bool commit_failure = false; 278 279 if (current_chunk() != NULL) { 280 281 // Attempt to satisfy the allocation from the current chunk. 282 283 // If the current chunk is too small to hold the requested size, attempt to enlarge it. 284 // If that fails, retire the chunk. 285 if (current_chunk()->free_words() < raw_word_size) { 286 if (!attempt_enlarge_current_chunk(raw_word_size)) { 287 current_chunk_too_small = true; 288 } else { 289 DEBUG_ONLY(InternalStats::inc_num_chunks_enlarged();) 290 UL(debug, "enlarged chunk."); 291 } 292 } 293 294 // Commit the chunk far enough to hold the requested word size. If that fails, we 295 // hit a limit (either GC threshold or MaxMetaspaceSize). In that case retire the 296 // chunk. 297 if (!current_chunk_too_small) { 298 if (!current_chunk()->ensure_committed_additional(raw_word_size)) { 299 UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", raw_word_size); 300 commit_failure = true; 301 } 302 } 303 304 // Allocate from the current chunk. This should work now. 305 if (!current_chunk_too_small && !commit_failure) { 306 p = current_chunk()->allocate(raw_word_size); 307 assert(p != NULL, "Allocation from chunk failed."); 308 } 309 } 310 311 if (p == NULL) { 312 // If we are here, we either had no current chunk to begin with or it was deemed insufficient. 313 assert(current_chunk() == NULL || 314 current_chunk_too_small || commit_failure, "Sanity"); 315 316 Metachunk* new_chunk = allocate_new_chunk(raw_word_size); 317 if (new_chunk != NULL) { 318 UL2(debug, "allocated new chunk " METACHUNK_FORMAT " for requested word size " SIZE_FORMAT ".", 319 METACHUNK_FORMAT_ARGS(new_chunk), requested_word_size); 320 321 assert(new_chunk->free_below_committed_words() >= raw_word_size, "Sanity"); 322 if (Settings::new_chunks_are_fully_committed()) { 323 assert(new_chunk->is_fully_committed(), "Chunk should be fully committed."); 324 } 325 326 // We have a new chunk. Before making it the current chunk, retire the old one. 327 if (current_chunk() != NULL) { 328 salvage_chunk(current_chunk()); 329 DEBUG_ONLY(InternalStats::inc_num_chunks_retired();) 330 } 331 332 _chunks.add(new_chunk); 333 334 // Now, allocate from that chunk. That should work. 335 p = current_chunk()->allocate(raw_word_size); 336 assert(p != NULL, "Allocation from chunk failed."); 337 } else { 338 UL2(info, "failed to allocate new chunk for requested word size " SIZE_FORMAT ".", requested_word_size); 339 } 340 } 341 342 if (p == NULL) { 343 InternalStats::inc_num_allocs_failed_limit(); 344 } else { 345 DEBUG_ONLY(InternalStats::inc_num_allocs();) 346 _total_used_words_counter->increment_by(raw_word_size); 347 } 348 349 SOMETIMES(verify_locked();) 350 351 if (p == NULL) { 352 UL(info, "allocation failed, returned NULL."); 353 } else { 354 UL2(trace, "after allocation: %u chunk(s), current:" METACHUNK_FULL_FORMAT, 355 _chunks.count(), METACHUNK_FULL_FORMAT_ARGS(current_chunk())); 356 UL2(trace, "returning " PTR_FORMAT ".", p2i(p)); 357 } 358 return p; 359 } 360 361 // Prematurely returns a metaspace allocation to the _block_freelists 362 // because it is not needed anymore (requires CLD lock to be active). 363 void MetaspaceArena::deallocate_locked(MetaWord* p, size_t word_size) { 364 assert_lock_strong(lock()); 365 // At this point a current chunk must exist since we only deallocate if we did allocate before. 366 assert(current_chunk() != NULL, "stray deallocation?"); 367 assert(is_valid_area(p, word_size), 368 "Pointer range not part of this Arena and cannot be deallocated: (" PTR_FORMAT ".." PTR_FORMAT ").", 369 p2i(p), p2i(p + word_size)); 370 371 UL2(trace, "deallocating " PTR_FORMAT ", word size: " SIZE_FORMAT ".", 372 p2i(p), word_size); 373 374 size_t raw_word_size = get_raw_word_size_for_requested_word_size(word_size, _alignment_words); 375 add_allocation_to_fbl(p, raw_word_size); 376 377 DEBUG_ONLY(verify_locked();) 378 } 379 380 // Prematurely returns a metaspace allocation to the _block_freelists because it is not 381 // needed anymore. 382 void MetaspaceArena::deallocate(MetaWord* p, size_t word_size) { 383 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); 384 deallocate_locked(p, word_size); 385 } 386 387 // Update statistics. This walks all in-use chunks. 388 void MetaspaceArena::add_to_statistics(ArenaStats* out) const { 389 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); 390 391 for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) { 392 InUseChunkStats& ucs = out->_stats[c->level()]; 393 ucs._num++; 394 ucs._word_size += c->word_size(); 395 ucs._committed_words += c->committed_words(); 396 ucs._used_words += c->used_words(); 397 // Note: for free and waste, we only count what's committed. 398 if (c == current_chunk()) { 399 ucs._free_words += c->free_below_committed_words(); 400 } else { 401 ucs._waste_words += c->free_below_committed_words(); 402 } 403 } 404 405 if (_fbl != NULL) { 406 out->_free_blocks_num += _fbl->count(); 407 out->_free_blocks_word_size += _fbl->total_size(); 408 } 409 410 SOMETIMES(out->verify();) 411 } 412 413 // Convenience method to get the most important usage statistics. 414 // For deeper analysis use add_to_statistics(). 415 void MetaspaceArena::usage_numbers(size_t* p_used_words, size_t* p_committed_words, size_t* p_capacity_words) const { 416 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); 417 size_t used = 0, comm = 0, cap = 0; 418 for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) { 419 used += c->used_words(); 420 comm += c->committed_words(); 421 cap += c->word_size(); 422 } 423 if (p_used_words != NULL) { 424 *p_used_words = used; 425 } 426 if (p_committed_words != NULL) { 427 *p_committed_words = comm; 428 } 429 if (p_capacity_words != NULL) { 430 *p_capacity_words = cap; 431 } 432 } 433 434 #ifdef ASSERT 435 436 void MetaspaceArena::verify_locked() const { 437 assert_lock_strong(lock()); 438 assert(_growth_policy != NULL && _chunk_manager != NULL, "Sanity"); 439 _chunks.verify(); 440 if (_fbl != NULL) { 441 _fbl->verify(); 442 } 443 } 444 445 void MetaspaceArena::Fence::verify() const { 446 assert(_eye1 == EyeCatcher && _eye2 == EyeCatcher, 447 "Metaspace corruption: fence block at " PTR_FORMAT " broken.", p2i(this)); 448 } 449 450 void MetaspaceArena::verify_allocation_guards() const { 451 assert(Settings::use_allocation_guard(), "Don't call with guards disabled."); 452 for (const Fence* f = _first_fence; f != NULL; f = f->next()) { 453 f->verify(); 454 } 455 } 456 457 void MetaspaceArena::verify() const { 458 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); 459 verify_locked(); 460 } 461 462 // Returns true if the area indicated by pointer and size have actually been allocated 463 // from this arena. 464 bool MetaspaceArena::is_valid_area(MetaWord* p, size_t word_size) const { 465 assert(p != NULL && word_size > 0, "Sanity"); 466 bool found = false; 467 for (const Metachunk* c = _chunks.first(); c != NULL && !found; c = c->next()) { 468 assert(c->is_valid_committed_pointer(p) == 469 c->is_valid_committed_pointer(p + word_size - 1), "range intersects"); 470 found = c->is_valid_committed_pointer(p); 471 } 472 return found; 473 } 474 475 #endif // ASSERT 476 477 void MetaspaceArena::print_on(outputStream* st) const { 478 MutexLocker fcl(_lock, Mutex::_no_safepoint_check_flag); 479 print_on_locked(st); 480 } 481 482 void MetaspaceArena::print_on_locked(outputStream* st) const { 483 assert_lock_strong(_lock); 484 st->print_cr("sm %s: %d chunks, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT, _name, 485 _chunks.count(), _chunks.calc_word_size(), _chunks.calc_committed_word_size()); 486 _chunks.print_on(st); 487 st->cr(); 488 st->print_cr("growth-policy " PTR_FORMAT ", alignment %d, lock " PTR_FORMAT ", cm " PTR_FORMAT ", fbl " PTR_FORMAT, 489 p2i(_growth_policy), _alignment_words * BytesPerWord, p2i(_lock), p2i(_chunk_manager), p2i(_fbl)); 490 } 491 492 } // namespace metaspace 493