1 /* 2 * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/classLoaderDataGraph.hpp" 27 #include "classfile/metadataOnStackMark.hpp" 28 #include "classfile/stringTable.hpp" 29 #include "code/codeCache.hpp" 30 #include "code/icBuffer.hpp" 31 #include "compiler/oopMap.hpp" 32 #include "gc/g1/g1Allocator.inline.hpp" 33 #include "gc/g1/g1Arguments.hpp" 34 #include "gc/g1/g1BarrierSet.hpp" 35 #include "gc/g1/g1BatchedTask.hpp" 36 #include "gc/g1/g1CollectedHeap.inline.hpp" 37 #include "gc/g1/g1CollectionSet.hpp" 38 #include "gc/g1/g1CollectionSetCandidates.hpp" 39 #include "gc/g1/g1CollectorState.hpp" 40 #include "gc/g1/g1ConcurrentRefine.hpp" 41 #include "gc/g1/g1ConcurrentRefineThread.hpp" 42 #include "gc/g1/g1ConcurrentMarkThread.inline.hpp" 43 #include "gc/g1/g1DirtyCardQueue.hpp" 44 #include "gc/g1/g1EvacStats.inline.hpp" 45 #include "gc/g1/g1FullCollector.hpp" 46 #include "gc/g1/g1GCCounters.hpp" 47 #include "gc/g1/g1GCParPhaseTimesTracker.hpp" 48 #include "gc/g1/g1GCPhaseTimes.hpp" 49 #include "gc/g1/g1GCPauseType.hpp" 50 #include "gc/g1/g1HeapSizingPolicy.hpp" 51 #include "gc/g1/g1HeapTransition.hpp" 52 #include "gc/g1/g1HeapVerifier.hpp" 53 #include "gc/g1/g1HotCardCache.hpp" 54 #include "gc/g1/g1InitLogger.hpp" 55 #include "gc/g1/g1MemoryPool.hpp" 56 #include "gc/g1/g1OopClosures.inline.hpp" 57 #include "gc/g1/g1ParallelCleaning.hpp" 58 #include "gc/g1/g1ParScanThreadState.inline.hpp" 59 #include "gc/g1/g1PeriodicGCTask.hpp" 60 #include "gc/g1/g1Policy.hpp" 61 #include "gc/g1/g1RedirtyCardsQueue.hpp" 62 #include "gc/g1/g1RegionToSpaceMapper.hpp" 63 #include "gc/g1/g1RemSet.hpp" 64 #include "gc/g1/g1RootClosures.hpp" 65 #include "gc/g1/g1RootProcessor.hpp" 66 #include "gc/g1/g1SATBMarkQueueSet.hpp" 67 #include "gc/g1/g1SegmentedArrayFreeMemoryTask.hpp" 68 #include "gc/g1/g1ServiceThread.hpp" 69 #include "gc/g1/g1ThreadLocalData.hpp" 70 #include "gc/g1/g1Trace.hpp" 71 #include "gc/g1/g1UncommitRegionTask.hpp" 72 #include "gc/g1/g1VMOperations.hpp" 73 #include "gc/g1/g1YoungCollector.hpp" 74 #include "gc/g1/g1YoungGCEvacFailureInjector.hpp" 75 #include "gc/g1/heapRegion.inline.hpp" 76 #include "gc/g1/heapRegionRemSet.inline.hpp" 77 #include "gc/g1/heapRegionSet.inline.hpp" 78 #include "gc/shared/concurrentGCBreakpoints.hpp" 79 #include "gc/shared/gcBehaviours.hpp" 80 #include "gc/shared/gcHeapSummary.hpp" 81 #include "gc/shared/gcId.hpp" 82 #include "gc/shared/gcLocker.hpp" 83 #include "gc/shared/gcTimer.hpp" 84 #include "gc/shared/gcTraceTime.inline.hpp" 85 #include "gc/shared/generationSpec.hpp" 86 #include "gc/shared/isGCActiveMark.hpp" 87 #include "gc/shared/locationPrinter.inline.hpp" 88 #include "gc/shared/oopStorageParState.hpp" 89 #include "gc/shared/preservedMarks.inline.hpp" 90 #include "gc/shared/referenceProcessor.inline.hpp" 91 #include "gc/shared/suspendibleThreadSet.hpp" 92 #include "gc/shared/taskqueue.inline.hpp" 93 #include "gc/shared/taskTerminator.hpp" 94 #include "gc/shared/tlab_globals.hpp" 95 #include "gc/shared/workerPolicy.hpp" 96 #include "gc/shared/weakProcessor.inline.hpp" 97 #include "logging/log.hpp" 98 #include "memory/allocation.hpp" 99 #include "memory/heapInspection.hpp" 100 #include "memory/iterator.hpp" 101 #include "memory/metaspaceUtils.hpp" 102 #include "memory/resourceArea.hpp" 103 #include "memory/universe.hpp" 104 #include "oops/access.inline.hpp" 105 #include "oops/compressedOops.inline.hpp" 106 #include "oops/oop.inline.hpp" 107 #include "runtime/atomic.hpp" 108 #include "runtime/handles.inline.hpp" 109 #include "runtime/init.hpp" 110 #include "runtime/java.hpp" 111 #include "runtime/orderAccess.hpp" 112 #include "runtime/threadSMR.hpp" 113 #include "runtime/vmThread.hpp" 114 #include "utilities/align.hpp" 115 #include "utilities/autoRestore.hpp" 116 #include "utilities/bitMap.inline.hpp" 117 #include "utilities/globalDefinitions.hpp" 118 #include "utilities/stack.inline.hpp" 119 120 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0; 121 122 // INVARIANTS/NOTES 123 // 124 // All allocation activity covered by the G1CollectedHeap interface is 125 // serialized by acquiring the HeapLock. This happens in mem_allocate 126 // and allocate_new_tlab, which are the "entry" points to the 127 // allocation code from the rest of the JVM. (Note that this does not 128 // apply to TLAB allocation, which is not part of this interface: it 129 // is done by clients of this interface.) 130 131 void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) { 132 HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions); 133 } 134 135 void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { 136 // The from card cache is not the memory that is actually committed. So we cannot 137 // take advantage of the zero_filled parameter. 138 reset_from_card_cache(start_idx, num_regions); 139 } 140 141 void G1CollectedHeap::run_batch_task(G1BatchedTask* cl) { 142 uint num_workers = MAX2(1u, MIN2(cl->num_workers_estimate(), workers()->active_workers())); 143 cl->set_max_workers(num_workers); 144 workers()->run_task(cl, num_workers); 145 } 146 147 HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, 148 MemRegion mr) { 149 return new HeapRegion(hrs_index, bot(), mr, &_card_set_config); 150 } 151 152 // Private methods. 153 154 HeapRegion* G1CollectedHeap::new_region(size_t word_size, 155 HeapRegionType type, 156 bool do_expand, 157 uint node_index) { 158 assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords, 159 "the only time we use this to allocate a humongous region is " 160 "when we are allocating a single humongous region"); 161 162 HeapRegion* res = _hrm.allocate_free_region(type, node_index); 163 164 if (res == NULL && do_expand) { 165 // Currently, only attempts to allocate GC alloc regions set 166 // do_expand to true. So, we should only reach here during a 167 // safepoint. 168 assert(SafepointSynchronize::is_at_safepoint(), "invariant"); 169 170 log_debug(gc, ergo, heap)("Attempt heap expansion (region allocation request failed). Allocation request: " SIZE_FORMAT "B", 171 word_size * HeapWordSize); 172 173 assert(word_size * HeapWordSize < HeapRegion::GrainBytes, 174 "This kind of expansion should never be more than one region. Size: " SIZE_FORMAT, 175 word_size * HeapWordSize); 176 if (expand_single_region(node_index)) { 177 // Given that expand_single_region() succeeded in expanding the heap, and we 178 // always expand the heap by an amount aligned to the heap 179 // region size, the free list should in theory not be empty. 180 // In either case allocate_free_region() will check for NULL. 181 res = _hrm.allocate_free_region(type, node_index); 182 } 183 } 184 return res; 185 } 186 187 HeapWord* 188 G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr, 189 uint num_regions, 190 size_t word_size) { 191 assert(first_hr != NULL, "pre-condition"); 192 assert(is_humongous(word_size), "word_size should be humongous"); 193 assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); 194 195 // Index of last region in the series. 196 uint first = first_hr->hrm_index(); 197 uint last = first + num_regions - 1; 198 199 // We need to initialize the region(s) we just discovered. This is 200 // a bit tricky given that it can happen concurrently with 201 // refinement threads refining cards on these regions and 202 // potentially wanting to refine the BOT as they are scanning 203 // those cards (this can happen shortly after a cleanup; see CR 204 // 6991377). So we have to set up the region(s) carefully and in 205 // a specific order. 206 207 // The word size sum of all the regions we will allocate. 208 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; 209 assert(word_size <= word_size_sum, "sanity"); 210 211 // The passed in hr will be the "starts humongous" region. The header 212 // of the new object will be placed at the bottom of this region. 213 HeapWord* new_obj = first_hr->bottom(); 214 // This will be the new top of the new object. 215 HeapWord* obj_top = new_obj + word_size; 216 217 // First, we need to zero the header of the space that we will be 218 // allocating. When we update top further down, some refinement 219 // threads might try to scan the region. By zeroing the header we 220 // ensure that any thread that will try to scan the region will 221 // come across the zero klass word and bail out. 222 // 223 // NOTE: It would not have been correct to have used 224 // CollectedHeap::fill_with_object() and make the space look like 225 // an int array. The thread that is doing the allocation will 226 // later update the object header to a potentially different array 227 // type and, for a very short period of time, the klass and length 228 // fields will be inconsistent. This could cause a refinement 229 // thread to calculate the object size incorrectly. 230 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); 231 232 // Next, pad out the unused tail of the last region with filler 233 // objects, for improved usage accounting. 234 // How many words we use for filler objects. 235 size_t word_fill_size = word_size_sum - word_size; 236 237 // How many words memory we "waste" which cannot hold a filler object. 238 size_t words_not_fillable = 0; 239 240 if (word_fill_size >= min_fill_size()) { 241 fill_with_objects(obj_top, word_fill_size); 242 } else if (word_fill_size > 0) { 243 // We have space to fill, but we cannot fit an object there. 244 words_not_fillable = word_fill_size; 245 word_fill_size = 0; 246 } 247 248 // We will set up the first region as "starts humongous". This 249 // will also update the BOT covering all the regions to reflect 250 // that there is a single object that starts at the bottom of the 251 // first region. 252 first_hr->set_starts_humongous(obj_top, word_fill_size); 253 _policy->remset_tracker()->update_at_allocate(first_hr); 254 // Then, if there are any, we will set up the "continues 255 // humongous" regions. 256 HeapRegion* hr = NULL; 257 for (uint i = first + 1; i <= last; ++i) { 258 hr = region_at(i); 259 hr->set_continues_humongous(first_hr); 260 _policy->remset_tracker()->update_at_allocate(hr); 261 } 262 263 // Up to this point no concurrent thread would have been able to 264 // do any scanning on any region in this series. All the top 265 // fields still point to bottom, so the intersection between 266 // [bottom,top] and [card_start,card_end] will be empty. Before we 267 // update the top fields, we'll do a storestore to make sure that 268 // no thread sees the update to top before the zeroing of the 269 // object header and the BOT initialization. 270 OrderAccess::storestore(); 271 272 // Now, we will update the top fields of the "continues humongous" 273 // regions except the last one. 274 for (uint i = first; i < last; ++i) { 275 hr = region_at(i); 276 hr->set_top(hr->end()); 277 } 278 279 hr = region_at(last); 280 // If we cannot fit a filler object, we must set top to the end 281 // of the humongous object, otherwise we cannot iterate the heap 282 // and the BOT will not be complete. 283 hr->set_top(hr->end() - words_not_fillable); 284 285 assert(hr->bottom() < obj_top && obj_top <= hr->end(), 286 "obj_top should be in last region"); 287 288 _verifier->check_bitmaps("Humongous Region Allocation", first_hr); 289 290 assert(words_not_fillable == 0 || 291 first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(), 292 "Miscalculation in humongous allocation"); 293 294 increase_used((word_size_sum - words_not_fillable) * HeapWordSize); 295 296 for (uint i = first; i <= last; ++i) { 297 hr = region_at(i); 298 _humongous_set.add(hr); 299 _hr_printer.alloc(hr); 300 } 301 302 return new_obj; 303 } 304 305 size_t G1CollectedHeap::humongous_obj_size_in_regions(size_t word_size) { 306 assert(is_humongous(word_size), "Object of size " SIZE_FORMAT " must be humongous here", word_size); 307 return align_up(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; 308 } 309 310 // If could fit into free regions w/o expansion, try. 311 // Otherwise, if can expand, do so. 312 // Otherwise, if using ex regions might help, try with ex given back. 313 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) { 314 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); 315 316 _verifier->verify_region_sets_optional(); 317 318 uint obj_regions = (uint) humongous_obj_size_in_regions(word_size); 319 320 // Policy: First try to allocate a humongous object in the free list. 321 HeapRegion* humongous_start = _hrm.allocate_humongous(obj_regions); 322 if (humongous_start == NULL) { 323 // Policy: We could not find enough regions for the humongous object in the 324 // free list. Look through the heap to find a mix of free and uncommitted regions. 325 // If so, expand the heap and allocate the humongous object. 326 humongous_start = _hrm.expand_and_allocate_humongous(obj_regions); 327 if (humongous_start != NULL) { 328 // We managed to find a region by expanding the heap. 329 log_debug(gc, ergo, heap)("Heap expansion (humongous allocation request). Allocation request: " SIZE_FORMAT "B", 330 word_size * HeapWordSize); 331 policy()->record_new_heap_size(num_regions()); 332 } else { 333 // Policy: Potentially trigger a defragmentation GC. 334 } 335 } 336 337 HeapWord* result = NULL; 338 if (humongous_start != NULL) { 339 result = humongous_obj_allocate_initialize_regions(humongous_start, obj_regions, word_size); 340 assert(result != NULL, "it should always return a valid result"); 341 342 // A successful humongous object allocation changes the used space 343 // information of the old generation so we need to recalculate the 344 // sizes and update the jstat counters here. 345 monitoring_support()->update_sizes(); 346 } 347 348 _verifier->verify_region_sets_optional(); 349 350 return result; 351 } 352 353 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t min_size, 354 size_t requested_size, 355 size_t* actual_size) { 356 assert_heap_not_locked_and_not_at_safepoint(); 357 assert(!is_humongous(requested_size), "we do not allow humongous TLABs"); 358 359 return attempt_allocation(min_size, requested_size, actual_size); 360 } 361 362 HeapWord* 363 G1CollectedHeap::mem_allocate(size_t word_size, 364 bool* gc_overhead_limit_was_exceeded) { 365 assert_heap_not_locked_and_not_at_safepoint(); 366 367 if (is_humongous(word_size)) { 368 return attempt_allocation_humongous(word_size); 369 } 370 size_t dummy = 0; 371 return attempt_allocation(word_size, word_size, &dummy); 372 } 373 374 HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size) { 375 ResourceMark rm; // For retrieving the thread names in log messages. 376 377 // Make sure you read the note in attempt_allocation_humongous(). 378 379 assert_heap_not_locked_and_not_at_safepoint(); 380 assert(!is_humongous(word_size), "attempt_allocation_slow() should not " 381 "be called for humongous allocation requests"); 382 383 // We should only get here after the first-level allocation attempt 384 // (attempt_allocation()) failed to allocate. 385 386 // We will loop until a) we manage to successfully perform the 387 // allocation or b) we successfully schedule a collection which 388 // fails to perform the allocation. b) is the only case when we'll 389 // return NULL. 390 HeapWord* result = NULL; 391 for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { 392 bool should_try_gc; 393 bool preventive_collection_required = false; 394 uint gc_count_before; 395 396 { 397 MutexLocker x(Heap_lock); 398 399 // Now that we have the lock, we first retry the allocation in case another 400 // thread changed the region while we were waiting to acquire the lock. 401 size_t actual_size; 402 result = _allocator->attempt_allocation(word_size, word_size, &actual_size); 403 if (result != NULL) { 404 return result; 405 } 406 407 preventive_collection_required = policy()->preventive_collection_required(1); 408 if (!preventive_collection_required) { 409 // We've already attempted a lock-free allocation above, so we don't want to 410 // do it again. Let's jump straight to replacing the active region. 411 result = _allocator->attempt_allocation_using_new_region(word_size); 412 if (result != NULL) { 413 return result; 414 } 415 416 // If the GCLocker is active and we are bound for a GC, try expanding young gen. 417 // This is different to when only GCLocker::needs_gc() is set: try to avoid 418 // waiting because the GCLocker is active to not wait too long. 419 if (GCLocker::is_active_and_needs_gc() && policy()->can_expand_young_list()) { 420 // No need for an ergo message here, can_expand_young_list() does this when 421 // it returns true. 422 result = _allocator->attempt_allocation_force(word_size); 423 if (result != NULL) { 424 return result; 425 } 426 } 427 } 428 429 // Only try a GC if the GCLocker does not signal the need for a GC. Wait until 430 // the GCLocker initiated GC has been performed and then retry. This includes 431 // the case when the GC Locker is not active but has not been performed. 432 should_try_gc = !GCLocker::needs_gc(); 433 // Read the GC count while still holding the Heap_lock. 434 gc_count_before = total_collections(); 435 } 436 437 if (should_try_gc) { 438 GCCause::Cause gc_cause = preventive_collection_required ? GCCause::_g1_preventive_collection 439 : GCCause::_g1_inc_collection_pause; 440 bool succeeded; 441 result = do_collection_pause(word_size, gc_count_before, &succeeded, gc_cause); 442 if (result != NULL) { 443 assert(succeeded, "only way to get back a non-NULL result"); 444 log_trace(gc, alloc)("%s: Successfully scheduled collection returning " PTR_FORMAT, 445 Thread::current()->name(), p2i(result)); 446 return result; 447 } 448 449 if (succeeded) { 450 // We successfully scheduled a collection which failed to allocate. No 451 // point in trying to allocate further. We'll just return NULL. 452 log_trace(gc, alloc)("%s: Successfully scheduled collection failing to allocate " 453 SIZE_FORMAT " words", Thread::current()->name(), word_size); 454 return NULL; 455 } 456 log_trace(gc, alloc)("%s: Unsuccessfully scheduled collection allocating " SIZE_FORMAT " words", 457 Thread::current()->name(), word_size); 458 } else { 459 // Failed to schedule a collection. 460 if (gclocker_retry_count > GCLockerRetryAllocationCount) { 461 log_warning(gc, alloc)("%s: Retried waiting for GCLocker too often allocating " 462 SIZE_FORMAT " words", Thread::current()->name(), word_size); 463 return NULL; 464 } 465 log_trace(gc, alloc)("%s: Stall until clear", Thread::current()->name()); 466 // The GCLocker is either active or the GCLocker initiated 467 // GC has not yet been performed. Stall until it is and 468 // then retry the allocation. 469 GCLocker::stall_until_clear(); 470 gclocker_retry_count += 1; 471 } 472 473 // We can reach here if we were unsuccessful in scheduling a 474 // collection (because another thread beat us to it) or if we were 475 // stalled due to the GC locker. In either can we should retry the 476 // allocation attempt in case another thread successfully 477 // performed a collection and reclaimed enough space. We do the 478 // first attempt (without holding the Heap_lock) here and the 479 // follow-on attempt will be at the start of the next loop 480 // iteration (after taking the Heap_lock). 481 size_t dummy = 0; 482 result = _allocator->attempt_allocation(word_size, word_size, &dummy); 483 if (result != NULL) { 484 return result; 485 } 486 487 // Give a warning if we seem to be looping forever. 488 if ((QueuedAllocationWarningCount > 0) && 489 (try_count % QueuedAllocationWarningCount == 0)) { 490 log_warning(gc, alloc)("%s: Retried allocation %u times for " SIZE_FORMAT " words", 491 Thread::current()->name(), try_count, word_size); 492 } 493 } 494 495 ShouldNotReachHere(); 496 return NULL; 497 } 498 499 void G1CollectedHeap::begin_archive_alloc_range(bool open) { 500 assert_at_safepoint_on_vm_thread(); 501 assert(_archive_allocator == nullptr, "should not be initialized"); 502 _archive_allocator = G1ArchiveAllocator::create_allocator(this, open); 503 } 504 505 bool G1CollectedHeap::is_archive_alloc_too_large(size_t word_size) { 506 // Allocations in archive regions cannot be of a size that would be considered 507 // humongous even for a minimum-sized region, because G1 region sizes/boundaries 508 // may be different at archive-restore time. 509 return word_size >= humongous_threshold_for(HeapRegion::min_region_size_in_words()); 510 } 511 512 HeapWord* G1CollectedHeap::archive_mem_allocate(size_t word_size) { 513 assert_at_safepoint_on_vm_thread(); 514 assert(_archive_allocator != nullptr, "_archive_allocator not initialized"); 515 if (is_archive_alloc_too_large(word_size)) { 516 return nullptr; 517 } 518 return _archive_allocator->archive_mem_allocate(word_size); 519 } 520 521 void G1CollectedHeap::end_archive_alloc_range(GrowableArray<MemRegion>* ranges, 522 size_t end_alignment_in_bytes) { 523 assert_at_safepoint_on_vm_thread(); 524 assert(_archive_allocator != nullptr, "_archive_allocator not initialized"); 525 526 // Call complete_archive to do the real work, filling in the MemRegion 527 // array with the archive regions. 528 _archive_allocator->complete_archive(ranges, end_alignment_in_bytes); 529 delete _archive_allocator; 530 _archive_allocator = nullptr; 531 } 532 533 bool G1CollectedHeap::check_archive_addresses(MemRegion* ranges, size_t count) { 534 assert(ranges != NULL, "MemRegion array NULL"); 535 assert(count != 0, "No MemRegions provided"); 536 MemRegion reserved = _hrm.reserved(); 537 for (size_t i = 0; i < count; i++) { 538 if (!reserved.contains(ranges[i].start()) || !reserved.contains(ranges[i].last())) { 539 return false; 540 } 541 } 542 return true; 543 } 544 545 bool G1CollectedHeap::alloc_archive_regions(MemRegion* ranges, 546 size_t count, 547 bool open) { 548 assert(!is_init_completed(), "Expect to be called at JVM init time"); 549 assert(ranges != NULL, "MemRegion array NULL"); 550 assert(count != 0, "No MemRegions provided"); 551 MutexLocker x(Heap_lock); 552 553 MemRegion reserved = _hrm.reserved(); 554 HeapWord* prev_last_addr = NULL; 555 HeapRegion* prev_last_region = NULL; 556 557 // Temporarily disable pretouching of heap pages. This interface is used 558 // when mmap'ing archived heap data in, so pre-touching is wasted. 559 FlagSetting fs(AlwaysPreTouch, false); 560 561 // For each specified MemRegion range, allocate the corresponding G1 562 // regions and mark them as archive regions. We expect the ranges 563 // in ascending starting address order, without overlap. 564 for (size_t i = 0; i < count; i++) { 565 MemRegion curr_range = ranges[i]; 566 HeapWord* start_address = curr_range.start(); 567 size_t word_size = curr_range.word_size(); 568 HeapWord* last_address = curr_range.last(); 569 size_t commits = 0; 570 571 guarantee(reserved.contains(start_address) && reserved.contains(last_address), 572 "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]", 573 p2i(start_address), p2i(last_address)); 574 guarantee(start_address > prev_last_addr, 575 "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT , 576 p2i(start_address), p2i(prev_last_addr)); 577 prev_last_addr = last_address; 578 579 // Check for ranges that start in the same G1 region in which the previous 580 // range ended, and adjust the start address so we don't try to allocate 581 // the same region again. If the current range is entirely within that 582 // region, skip it, just adjusting the recorded top. 583 HeapRegion* start_region = _hrm.addr_to_region(start_address); 584 if ((prev_last_region != NULL) && (start_region == prev_last_region)) { 585 start_address = start_region->end(); 586 if (start_address > last_address) { 587 increase_used(word_size * HeapWordSize); 588 start_region->set_top(last_address + 1); 589 continue; 590 } 591 start_region->set_top(start_address); 592 curr_range = MemRegion(start_address, last_address + 1); 593 start_region = _hrm.addr_to_region(start_address); 594 } 595 596 // Perform the actual region allocation, exiting if it fails. 597 // Then note how much new space we have allocated. 598 if (!_hrm.allocate_containing_regions(curr_range, &commits, workers())) { 599 return false; 600 } 601 increase_used(word_size * HeapWordSize); 602 if (commits != 0) { 603 log_debug(gc, ergo, heap)("Attempt heap expansion (allocate archive regions). Total size: " SIZE_FORMAT "B", 604 HeapRegion::GrainWords * HeapWordSize * commits); 605 606 } 607 608 // Mark each G1 region touched by the range as archive, add it to 609 // the old set, and set top. 610 HeapRegion* curr_region = _hrm.addr_to_region(start_address); 611 HeapRegion* last_region = _hrm.addr_to_region(last_address); 612 prev_last_region = last_region; 613 614 while (curr_region != NULL) { 615 assert(curr_region->is_empty() && !curr_region->is_pinned(), 616 "Region already in use (index %u)", curr_region->hrm_index()); 617 if (open) { 618 curr_region->set_open_archive(); 619 } else { 620 curr_region->set_closed_archive(); 621 } 622 _hr_printer.alloc(curr_region); 623 _archive_set.add(curr_region); 624 HeapWord* top; 625 HeapRegion* next_region; 626 if (curr_region != last_region) { 627 top = curr_region->end(); 628 next_region = _hrm.next_region_in_heap(curr_region); 629 } else { 630 top = last_address + 1; 631 next_region = NULL; 632 } 633 curr_region->set_top(top); 634 curr_region = next_region; 635 } 636 } 637 return true; 638 } 639 640 void G1CollectedHeap::fill_archive_regions(MemRegion* ranges, size_t count) { 641 assert(!is_init_completed(), "Expect to be called at JVM init time"); 642 assert(ranges != NULL, "MemRegion array NULL"); 643 assert(count != 0, "No MemRegions provided"); 644 MemRegion reserved = _hrm.reserved(); 645 HeapWord *prev_last_addr = NULL; 646 HeapRegion* prev_last_region = NULL; 647 648 // For each MemRegion, create filler objects, if needed, in the G1 regions 649 // that contain the address range. The address range actually within the 650 // MemRegion will not be modified. That is assumed to have been initialized 651 // elsewhere, probably via an mmap of archived heap data. 652 MutexLocker x(Heap_lock); 653 for (size_t i = 0; i < count; i++) { 654 HeapWord* start_address = ranges[i].start(); 655 HeapWord* last_address = ranges[i].last(); 656 657 assert(reserved.contains(start_address) && reserved.contains(last_address), 658 "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]", 659 p2i(start_address), p2i(last_address)); 660 assert(start_address > prev_last_addr, 661 "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT , 662 p2i(start_address), p2i(prev_last_addr)); 663 664 HeapRegion* start_region = _hrm.addr_to_region(start_address); 665 HeapRegion* last_region = _hrm.addr_to_region(last_address); 666 HeapWord* bottom_address = start_region->bottom(); 667 668 // Check for a range beginning in the same region in which the 669 // previous one ended. 670 if (start_region == prev_last_region) { 671 bottom_address = prev_last_addr + 1; 672 } 673 674 // Verify that the regions were all marked as archive regions by 675 // alloc_archive_regions. 676 HeapRegion* curr_region = start_region; 677 while (curr_region != NULL) { 678 guarantee(curr_region->is_archive(), 679 "Expected archive region at index %u", curr_region->hrm_index()); 680 if (curr_region != last_region) { 681 curr_region = _hrm.next_region_in_heap(curr_region); 682 } else { 683 curr_region = NULL; 684 } 685 } 686 687 prev_last_addr = last_address; 688 prev_last_region = last_region; 689 690 // Fill the memory below the allocated range with dummy object(s), 691 // if the region bottom does not match the range start, or if the previous 692 // range ended within the same G1 region, and there is a gap. 693 assert(start_address >= bottom_address, "bottom address should not be greater than start address"); 694 if (start_address > bottom_address) { 695 size_t fill_size = pointer_delta(start_address, bottom_address); 696 G1CollectedHeap::fill_with_objects(bottom_address, fill_size); 697 increase_used(fill_size * HeapWordSize); 698 } 699 } 700 } 701 702 inline HeapWord* G1CollectedHeap::attempt_allocation(size_t min_word_size, 703 size_t desired_word_size, 704 size_t* actual_word_size) { 705 assert_heap_not_locked_and_not_at_safepoint(); 706 assert(!is_humongous(desired_word_size), "attempt_allocation() should not " 707 "be called for humongous allocation requests"); 708 709 HeapWord* result = _allocator->attempt_allocation(min_word_size, desired_word_size, actual_word_size); 710 711 if (result == NULL) { 712 *actual_word_size = desired_word_size; 713 result = attempt_allocation_slow(desired_word_size); 714 } 715 716 assert_heap_not_locked(); 717 if (result != NULL) { 718 assert(*actual_word_size != 0, "Actual size must have been set here"); 719 dirty_young_block(result, *actual_word_size); 720 } else { 721 *actual_word_size = 0; 722 } 723 724 return result; 725 } 726 727 void G1CollectedHeap::populate_archive_regions_bot_part(MemRegion* ranges, size_t count) { 728 assert(!is_init_completed(), "Expect to be called at JVM init time"); 729 assert(ranges != NULL, "MemRegion array NULL"); 730 assert(count != 0, "No MemRegions provided"); 731 732 HeapWord* st = ranges[0].start(); 733 HeapWord* last = ranges[count-1].last(); 734 HeapRegion* hr_st = _hrm.addr_to_region(st); 735 HeapRegion* hr_last = _hrm.addr_to_region(last); 736 737 HeapRegion* hr_curr = hr_st; 738 while (hr_curr != NULL) { 739 hr_curr->update_bot(); 740 if (hr_curr != hr_last) { 741 hr_curr = _hrm.next_region_in_heap(hr_curr); 742 } else { 743 hr_curr = NULL; 744 } 745 } 746 } 747 748 void G1CollectedHeap::dealloc_archive_regions(MemRegion* ranges, size_t count) { 749 assert(!is_init_completed(), "Expect to be called at JVM init time"); 750 assert(ranges != NULL, "MemRegion array NULL"); 751 assert(count != 0, "No MemRegions provided"); 752 MemRegion reserved = _hrm.reserved(); 753 HeapWord* prev_last_addr = NULL; 754 HeapRegion* prev_last_region = NULL; 755 size_t size_used = 0; 756 uint shrink_count = 0; 757 758 // For each Memregion, free the G1 regions that constitute it, and 759 // notify mark-sweep that the range is no longer to be considered 'archive.' 760 MutexLocker x(Heap_lock); 761 for (size_t i = 0; i < count; i++) { 762 HeapWord* start_address = ranges[i].start(); 763 HeapWord* last_address = ranges[i].last(); 764 765 assert(reserved.contains(start_address) && reserved.contains(last_address), 766 "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]", 767 p2i(start_address), p2i(last_address)); 768 assert(start_address > prev_last_addr, 769 "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT , 770 p2i(start_address), p2i(prev_last_addr)); 771 size_used += ranges[i].byte_size(); 772 prev_last_addr = last_address; 773 774 HeapRegion* start_region = _hrm.addr_to_region(start_address); 775 HeapRegion* last_region = _hrm.addr_to_region(last_address); 776 777 // Check for ranges that start in the same G1 region in which the previous 778 // range ended, and adjust the start address so we don't try to free 779 // the same region again. If the current range is entirely within that 780 // region, skip it. 781 if (start_region == prev_last_region) { 782 start_address = start_region->end(); 783 if (start_address > last_address) { 784 continue; 785 } 786 start_region = _hrm.addr_to_region(start_address); 787 } 788 prev_last_region = last_region; 789 790 // After verifying that each region was marked as an archive region by 791 // alloc_archive_regions, set it free and empty and uncommit it. 792 HeapRegion* curr_region = start_region; 793 while (curr_region != NULL) { 794 guarantee(curr_region->is_archive(), 795 "Expected archive region at index %u", curr_region->hrm_index()); 796 uint curr_index = curr_region->hrm_index(); 797 _archive_set.remove(curr_region); 798 curr_region->set_free(); 799 curr_region->set_top(curr_region->bottom()); 800 if (curr_region != last_region) { 801 curr_region = _hrm.next_region_in_heap(curr_region); 802 } else { 803 curr_region = NULL; 804 } 805 806 _hrm.shrink_at(curr_index, 1); 807 shrink_count++; 808 } 809 } 810 811 if (shrink_count != 0) { 812 log_debug(gc, ergo, heap)("Attempt heap shrinking (archive regions). Total size: " SIZE_FORMAT "B", 813 HeapRegion::GrainWords * HeapWordSize * shrink_count); 814 // Explicit uncommit. 815 uncommit_regions(shrink_count); 816 } 817 decrease_used(size_used); 818 } 819 820 HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size) { 821 ResourceMark rm; // For retrieving the thread names in log messages. 822 823 // The structure of this method has a lot of similarities to 824 // attempt_allocation_slow(). The reason these two were not merged 825 // into a single one is that such a method would require several "if 826 // allocation is not humongous do this, otherwise do that" 827 // conditional paths which would obscure its flow. In fact, an early 828 // version of this code did use a unified method which was harder to 829 // follow and, as a result, it had subtle bugs that were hard to 830 // track down. So keeping these two methods separate allows each to 831 // be more readable. It will be good to keep these two in sync as 832 // much as possible. 833 834 assert_heap_not_locked_and_not_at_safepoint(); 835 assert(is_humongous(word_size), "attempt_allocation_humongous() " 836 "should only be called for humongous allocations"); 837 838 // Humongous objects can exhaust the heap quickly, so we should check if we 839 // need to start a marking cycle at each humongous object allocation. We do 840 // the check before we do the actual allocation. The reason for doing it 841 // before the allocation is that we avoid having to keep track of the newly 842 // allocated memory while we do a GC. 843 if (policy()->need_to_start_conc_mark("concurrent humongous allocation", 844 word_size)) { 845 collect(GCCause::_g1_humongous_allocation); 846 } 847 848 // We will loop until a) we manage to successfully perform the 849 // allocation or b) we successfully schedule a collection which 850 // fails to perform the allocation. b) is the only case when we'll 851 // return NULL. 852 HeapWord* result = NULL; 853 for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { 854 bool should_try_gc; 855 bool preventive_collection_required = false; 856 uint gc_count_before; 857 858 859 { 860 MutexLocker x(Heap_lock); 861 862 size_t size_in_regions = humongous_obj_size_in_regions(word_size); 863 preventive_collection_required = policy()->preventive_collection_required((uint)size_in_regions); 864 if (!preventive_collection_required) { 865 // Given that humongous objects are not allocated in young 866 // regions, we'll first try to do the allocation without doing a 867 // collection hoping that there's enough space in the heap. 868 result = humongous_obj_allocate(word_size); 869 if (result != NULL) { 870 policy()->old_gen_alloc_tracker()-> 871 add_allocated_humongous_bytes_since_last_gc(size_in_regions * HeapRegion::GrainBytes); 872 return result; 873 } 874 } 875 876 // Only try a GC if the GCLocker does not signal the need for a GC. Wait until 877 // the GCLocker initiated GC has been performed and then retry. This includes 878 // the case when the GC Locker is not active but has not been performed. 879 should_try_gc = !GCLocker::needs_gc(); 880 // Read the GC count while still holding the Heap_lock. 881 gc_count_before = total_collections(); 882 } 883 884 if (should_try_gc) { 885 GCCause::Cause gc_cause = preventive_collection_required ? GCCause::_g1_preventive_collection 886 : GCCause::_g1_humongous_allocation; 887 bool succeeded; 888 result = do_collection_pause(word_size, gc_count_before, &succeeded, gc_cause); 889 if (result != NULL) { 890 assert(succeeded, "only way to get back a non-NULL result"); 891 log_trace(gc, alloc)("%s: Successfully scheduled collection returning " PTR_FORMAT, 892 Thread::current()->name(), p2i(result)); 893 size_t size_in_regions = humongous_obj_size_in_regions(word_size); 894 policy()->old_gen_alloc_tracker()-> 895 record_collection_pause_humongous_allocation(size_in_regions * HeapRegion::GrainBytes); 896 return result; 897 } 898 899 if (succeeded) { 900 // We successfully scheduled a collection which failed to allocate. No 901 // point in trying to allocate further. We'll just return NULL. 902 log_trace(gc, alloc)("%s: Successfully scheduled collection failing to allocate " 903 SIZE_FORMAT " words", Thread::current()->name(), word_size); 904 return NULL; 905 } 906 log_trace(gc, alloc)("%s: Unsuccessfully scheduled collection allocating " SIZE_FORMAT "", 907 Thread::current()->name(), word_size); 908 } else { 909 // Failed to schedule a collection. 910 if (gclocker_retry_count > GCLockerRetryAllocationCount) { 911 log_warning(gc, alloc)("%s: Retried waiting for GCLocker too often allocating " 912 SIZE_FORMAT " words", Thread::current()->name(), word_size); 913 return NULL; 914 } 915 log_trace(gc, alloc)("%s: Stall until clear", Thread::current()->name()); 916 // The GCLocker is either active or the GCLocker initiated 917 // GC has not yet been performed. Stall until it is and 918 // then retry the allocation. 919 GCLocker::stall_until_clear(); 920 gclocker_retry_count += 1; 921 } 922 923 924 // We can reach here if we were unsuccessful in scheduling a 925 // collection (because another thread beat us to it) or if we were 926 // stalled due to the GC locker. In either can we should retry the 927 // allocation attempt in case another thread successfully 928 // performed a collection and reclaimed enough space. 929 // Humongous object allocation always needs a lock, so we wait for the retry 930 // in the next iteration of the loop, unlike for the regular iteration case. 931 // Give a warning if we seem to be looping forever. 932 933 if ((QueuedAllocationWarningCount > 0) && 934 (try_count % QueuedAllocationWarningCount == 0)) { 935 log_warning(gc, alloc)("%s: Retried allocation %u times for " SIZE_FORMAT " words", 936 Thread::current()->name(), try_count, word_size); 937 } 938 } 939 940 ShouldNotReachHere(); 941 return NULL; 942 } 943 944 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, 945 bool expect_null_mutator_alloc_region) { 946 assert_at_safepoint_on_vm_thread(); 947 assert(!_allocator->has_mutator_alloc_region() || !expect_null_mutator_alloc_region, 948 "the current alloc region was unexpectedly found to be non-NULL"); 949 950 if (!is_humongous(word_size)) { 951 return _allocator->attempt_allocation_locked(word_size); 952 } else { 953 HeapWord* result = humongous_obj_allocate(word_size); 954 if (result != NULL && policy()->need_to_start_conc_mark("STW humongous allocation")) { 955 collector_state()->set_initiate_conc_mark_if_possible(true); 956 } 957 return result; 958 } 959 960 ShouldNotReachHere(); 961 } 962 963 class PostCompactionPrinterClosure: public HeapRegionClosure { 964 private: 965 G1HRPrinter* _hr_printer; 966 public: 967 bool do_heap_region(HeapRegion* hr) { 968 assert(!hr->is_young(), "not expecting to find young regions"); 969 _hr_printer->post_compaction(hr); 970 return false; 971 } 972 973 PostCompactionPrinterClosure(G1HRPrinter* hr_printer) 974 : _hr_printer(hr_printer) { } 975 }; 976 977 void G1CollectedHeap::print_heap_after_full_collection() { 978 // Post collection region logging. 979 // We should do this after we potentially resize the heap so 980 // that all the COMMIT / UNCOMMIT events are generated before 981 // the compaction events. 982 if (_hr_printer.is_active()) { 983 PostCompactionPrinterClosure cl(hr_printer()); 984 heap_region_iterate(&cl); 985 } 986 } 987 988 void G1CollectedHeap::abort_concurrent_cycle() { 989 // If we start the compaction before the CM threads finish 990 // scanning the root regions we might trip them over as we'll 991 // be moving objects / updating references. So let's wait until 992 // they are done. By telling them to abort, they should complete 993 // early. 994 _cm->root_regions()->abort(); 995 _cm->root_regions()->wait_until_scan_finished(); 996 997 // Disable discovery and empty the discovered lists 998 // for the CM ref processor. 999 _ref_processor_cm->disable_discovery(); 1000 _ref_processor_cm->abandon_partial_discovery(); 1001 _ref_processor_cm->verify_no_references_recorded(); 1002 1003 // Abandon current iterations of concurrent marking and concurrent 1004 // refinement, if any are in progress. 1005 concurrent_mark()->concurrent_cycle_abort(); 1006 } 1007 1008 void G1CollectedHeap::prepare_heap_for_full_collection() { 1009 // Make sure we'll choose a new allocation region afterwards. 1010 _allocator->release_mutator_alloc_regions(); 1011 _allocator->abandon_gc_alloc_regions(); 1012 1013 // We may have added regions to the current incremental collection 1014 // set between the last GC or pause and now. We need to clear the 1015 // incremental collection set and then start rebuilding it afresh 1016 // after this full GC. 1017 abandon_collection_set(collection_set()); 1018 1019 _hrm.remove_all_free_regions(); 1020 } 1021 1022 void G1CollectedHeap::verify_before_full_collection(bool explicit_gc) { 1023 assert(!GCCause::is_user_requested_gc(gc_cause()) || explicit_gc, "invariant"); 1024 assert_used_and_recalculate_used_equal(this); 1025 if (!VerifyBeforeGC) { 1026 return; 1027 } 1028 _verifier->verify_region_sets_optional(); 1029 _verifier->verify_before_gc(G1HeapVerifier::G1VerifyFull); 1030 _verifier->check_bitmaps("Full GC Start"); 1031 } 1032 1033 void G1CollectedHeap::prepare_heap_for_mutators() { 1034 // Delete metaspaces for unloaded class loaders and clean up loader_data graph 1035 ClassLoaderDataGraph::purge(/*at_safepoint*/true); 1036 DEBUG_ONLY(MetaspaceUtils::verify();) 1037 1038 // Prepare heap for normal collections. 1039 assert(num_free_regions() == 0, "we should not have added any free regions"); 1040 rebuild_region_sets(false /* free_list_only */); 1041 abort_refinement(); 1042 resize_heap_if_necessary(); 1043 uncommit_regions_if_necessary(); 1044 1045 // Rebuild the code root lists for each region 1046 rebuild_code_roots(); 1047 1048 // Purge code root memory 1049 purge_code_root_memory(); 1050 1051 // Start a new incremental collection set for the next pause 1052 start_new_collection_set(); 1053 1054 _allocator->init_mutator_alloc_regions(); 1055 1056 // Post collection state updates. 1057 MetaspaceGC::compute_new_size(); 1058 } 1059 1060 void G1CollectedHeap::abort_refinement() { 1061 if (_hot_card_cache->use_cache()) { 1062 _hot_card_cache->reset_hot_cache(); 1063 } 1064 1065 // Discard all remembered set updates and reset refinement statistics. 1066 G1BarrierSet::dirty_card_queue_set().abandon_logs(); 1067 assert(G1BarrierSet::dirty_card_queue_set().num_cards() == 0, 1068 "DCQS should be empty"); 1069 concurrent_refine()->get_and_reset_refinement_stats(); 1070 } 1071 1072 void G1CollectedHeap::verify_after_full_collection() { 1073 if (!VerifyAfterGC) { 1074 return; 1075 } 1076 _hrm.verify_optional(); 1077 _verifier->verify_region_sets_optional(); 1078 _verifier->verify_after_gc(G1HeapVerifier::G1VerifyFull); 1079 1080 // This call implicitly verifies that the next bitmap is clear after Full GC. 1081 _verifier->check_bitmaps("Full GC End"); 1082 1083 // At this point there should be no regions in the 1084 // entire heap tagged as young. 1085 assert(check_young_list_empty(), "young list should be empty at this point"); 1086 1087 // Note: since we've just done a full GC, concurrent 1088 // marking is no longer active. Therefore we need not 1089 // re-enable reference discovery for the CM ref processor. 1090 // That will be done at the start of the next marking cycle. 1091 // We also know that the STW processor should no longer 1092 // discover any new references. 1093 assert(!_ref_processor_stw->discovery_enabled(), "Postcondition"); 1094 assert(!_ref_processor_cm->discovery_enabled(), "Postcondition"); 1095 _ref_processor_stw->verify_no_references_recorded(); 1096 _ref_processor_cm->verify_no_references_recorded(); 1097 } 1098 1099 bool G1CollectedHeap::do_full_collection(bool explicit_gc, 1100 bool clear_all_soft_refs, 1101 bool do_maximal_compaction) { 1102 assert_at_safepoint_on_vm_thread(); 1103 1104 if (GCLocker::check_active_before_gc()) { 1105 // Full GC was not completed. 1106 return false; 1107 } 1108 1109 const bool do_clear_all_soft_refs = clear_all_soft_refs || 1110 soft_ref_policy()->should_clear_all_soft_refs(); 1111 1112 G1FullGCMark gc_mark; 1113 GCTraceTime(Info, gc) tm("Pause Full", NULL, gc_cause(), true); 1114 G1FullCollector collector(this, explicit_gc, do_clear_all_soft_refs, do_maximal_compaction); 1115 1116 collector.prepare_collection(); 1117 collector.collect(); 1118 collector.complete_collection(); 1119 1120 // Full collection was successfully completed. 1121 return true; 1122 } 1123 1124 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) { 1125 // Currently, there is no facility in the do_full_collection(bool) API to notify 1126 // the caller that the collection did not succeed (e.g., because it was locked 1127 // out by the GC locker). So, right now, we'll ignore the return value. 1128 // When clear_all_soft_refs is set we want to do a maximal compaction 1129 // not leaving any dead wood. 1130 bool do_maximal_compaction = clear_all_soft_refs; 1131 bool dummy = do_full_collection(true, /* explicit_gc */ 1132 clear_all_soft_refs, 1133 do_maximal_compaction); 1134 } 1135 1136 bool G1CollectedHeap::upgrade_to_full_collection() { 1137 GCCauseSetter compaction(this, GCCause::_g1_compaction_pause); 1138 log_info(gc, ergo)("Attempting full compaction clearing soft references"); 1139 bool success = do_full_collection(false /* explicit gc */, 1140 true /* clear_all_soft_refs */, 1141 false /* do_maximal_compaction */); 1142 // do_full_collection only fails if blocked by GC locker and that can't 1143 // be the case here since we only call this when already completed one gc. 1144 assert(success, "invariant"); 1145 return success; 1146 } 1147 1148 void G1CollectedHeap::resize_heap_if_necessary() { 1149 assert_at_safepoint_on_vm_thread(); 1150 1151 bool should_expand; 1152 size_t resize_amount = _heap_sizing_policy->full_collection_resize_amount(should_expand); 1153 1154 if (resize_amount == 0) { 1155 return; 1156 } else if (should_expand) { 1157 expand(resize_amount, _workers); 1158 } else { 1159 shrink(resize_amount); 1160 } 1161 } 1162 1163 HeapWord* G1CollectedHeap::satisfy_failed_allocation_helper(size_t word_size, 1164 bool do_gc, 1165 bool maximal_compaction, 1166 bool expect_null_mutator_alloc_region, 1167 bool* gc_succeeded) { 1168 *gc_succeeded = true; 1169 // Let's attempt the allocation first. 1170 HeapWord* result = 1171 attempt_allocation_at_safepoint(word_size, 1172 expect_null_mutator_alloc_region); 1173 if (result != NULL) { 1174 return result; 1175 } 1176 1177 // In a G1 heap, we're supposed to keep allocation from failing by 1178 // incremental pauses. Therefore, at least for now, we'll favor 1179 // expansion over collection. (This might change in the future if we can 1180 // do something smarter than full collection to satisfy a failed alloc.) 1181 result = expand_and_allocate(word_size); 1182 if (result != NULL) { 1183 return result; 1184 } 1185 1186 if (do_gc) { 1187 GCCauseSetter compaction(this, GCCause::_g1_compaction_pause); 1188 // Expansion didn't work, we'll try to do a Full GC. 1189 // If maximal_compaction is set we clear all soft references and don't 1190 // allow any dead wood to be left on the heap. 1191 if (maximal_compaction) { 1192 log_info(gc, ergo)("Attempting maximal full compaction clearing soft references"); 1193 } else { 1194 log_info(gc, ergo)("Attempting full compaction"); 1195 } 1196 *gc_succeeded = do_full_collection(false, /* explicit_gc */ 1197 maximal_compaction /* clear_all_soft_refs */ , 1198 maximal_compaction /* do_maximal_compaction */); 1199 } 1200 1201 return NULL; 1202 } 1203 1204 HeapWord* G1CollectedHeap::satisfy_failed_allocation(size_t word_size, 1205 bool* succeeded) { 1206 assert_at_safepoint_on_vm_thread(); 1207 1208 // Attempts to allocate followed by Full GC. 1209 HeapWord* result = 1210 satisfy_failed_allocation_helper(word_size, 1211 true, /* do_gc */ 1212 false, /* maximum_collection */ 1213 false, /* expect_null_mutator_alloc_region */ 1214 succeeded); 1215 1216 if (result != NULL || !*succeeded) { 1217 return result; 1218 } 1219 1220 // Attempts to allocate followed by Full GC that will collect all soft references. 1221 result = satisfy_failed_allocation_helper(word_size, 1222 true, /* do_gc */ 1223 true, /* maximum_collection */ 1224 true, /* expect_null_mutator_alloc_region */ 1225 succeeded); 1226 1227 if (result != NULL || !*succeeded) { 1228 return result; 1229 } 1230 1231 // Attempts to allocate, no GC 1232 result = satisfy_failed_allocation_helper(word_size, 1233 false, /* do_gc */ 1234 false, /* maximum_collection */ 1235 true, /* expect_null_mutator_alloc_region */ 1236 succeeded); 1237 1238 if (result != NULL) { 1239 return result; 1240 } 1241 1242 assert(!soft_ref_policy()->should_clear_all_soft_refs(), 1243 "Flag should have been handled and cleared prior to this point"); 1244 1245 // What else? We might try synchronous finalization later. If the total 1246 // space available is large enough for the allocation, then a more 1247 // complete compaction phase than we've tried so far might be 1248 // appropriate. 1249 return NULL; 1250 } 1251 1252 // Attempting to expand the heap sufficiently 1253 // to support an allocation of the given "word_size". If 1254 // successful, perform the allocation and return the address of the 1255 // allocated block, or else "NULL". 1256 1257 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) { 1258 assert_at_safepoint_on_vm_thread(); 1259 1260 _verifier->verify_region_sets_optional(); 1261 1262 size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes); 1263 log_debug(gc, ergo, heap)("Attempt heap expansion (allocation request failed). Allocation request: " SIZE_FORMAT "B", 1264 word_size * HeapWordSize); 1265 1266 1267 if (expand(expand_bytes, _workers)) { 1268 _hrm.verify_optional(); 1269 _verifier->verify_region_sets_optional(); 1270 return attempt_allocation_at_safepoint(word_size, 1271 false /* expect_null_mutator_alloc_region */); 1272 } 1273 return NULL; 1274 } 1275 1276 bool G1CollectedHeap::expand(size_t expand_bytes, WorkerThreads* pretouch_workers, double* expand_time_ms) { 1277 size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); 1278 aligned_expand_bytes = align_up(aligned_expand_bytes, 1279 HeapRegion::GrainBytes); 1280 1281 log_debug(gc, ergo, heap)("Expand the heap. requested expansion amount: " SIZE_FORMAT "B expansion amount: " SIZE_FORMAT "B", 1282 expand_bytes, aligned_expand_bytes); 1283 1284 if (is_maximal_no_gc()) { 1285 log_debug(gc, ergo, heap)("Did not expand the heap (heap already fully expanded)"); 1286 return false; 1287 } 1288 1289 double expand_heap_start_time_sec = os::elapsedTime(); 1290 uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes); 1291 assert(regions_to_expand > 0, "Must expand by at least one region"); 1292 1293 uint expanded_by = _hrm.expand_by(regions_to_expand, pretouch_workers); 1294 if (expand_time_ms != NULL) { 1295 *expand_time_ms = (os::elapsedTime() - expand_heap_start_time_sec) * MILLIUNITS; 1296 } 1297 1298 if (expanded_by > 0) { 1299 size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes; 1300 assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition"); 1301 policy()->record_new_heap_size(num_regions()); 1302 } else { 1303 log_debug(gc, ergo, heap)("Did not expand the heap (heap expansion operation failed)"); 1304 1305 // The expansion of the virtual storage space was unsuccessful. 1306 // Let's see if it was because we ran out of swap. 1307 if (G1ExitOnExpansionFailure && 1308 _hrm.available() >= regions_to_expand) { 1309 // We had head room... 1310 vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion"); 1311 } 1312 } 1313 return expanded_by > 0; 1314 } 1315 1316 bool G1CollectedHeap::expand_single_region(uint node_index) { 1317 uint expanded_by = _hrm.expand_on_preferred_node(node_index); 1318 1319 if (expanded_by == 0) { 1320 assert(is_maximal_no_gc(), "Should be no regions left, available: %u", _hrm.available()); 1321 log_debug(gc, ergo, heap)("Did not expand the heap (heap already fully expanded)"); 1322 return false; 1323 } 1324 1325 policy()->record_new_heap_size(num_regions()); 1326 return true; 1327 } 1328 1329 void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { 1330 size_t aligned_shrink_bytes = 1331 ReservedSpace::page_align_size_down(shrink_bytes); 1332 aligned_shrink_bytes = align_down(aligned_shrink_bytes, 1333 HeapRegion::GrainBytes); 1334 uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes); 1335 1336 uint num_regions_removed = _hrm.shrink_by(num_regions_to_remove); 1337 size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes; 1338 1339 log_debug(gc, ergo, heap)("Shrink the heap. requested shrinking amount: " SIZE_FORMAT "B aligned shrinking amount: " SIZE_FORMAT "B attempted shrinking amount: " SIZE_FORMAT "B", 1340 shrink_bytes, aligned_shrink_bytes, shrunk_bytes); 1341 if (num_regions_removed > 0) { 1342 log_debug(gc, heap)("Uncommittable regions after shrink: %u", num_regions_removed); 1343 policy()->record_new_heap_size(num_regions()); 1344 } else { 1345 log_debug(gc, ergo, heap)("Did not expand the heap (heap shrinking operation failed)"); 1346 } 1347 } 1348 1349 void G1CollectedHeap::shrink(size_t shrink_bytes) { 1350 _verifier->verify_region_sets_optional(); 1351 1352 // We should only reach here at the end of a Full GC or during Remark which 1353 // means we should not not be holding to any GC alloc regions. The method 1354 // below will make sure of that and do any remaining clean up. 1355 _allocator->abandon_gc_alloc_regions(); 1356 1357 // Instead of tearing down / rebuilding the free lists here, we 1358 // could instead use the remove_all_pending() method on free_list to 1359 // remove only the ones that we need to remove. 1360 _hrm.remove_all_free_regions(); 1361 shrink_helper(shrink_bytes); 1362 rebuild_region_sets(true /* free_list_only */); 1363 1364 _hrm.verify_optional(); 1365 _verifier->verify_region_sets_optional(); 1366 } 1367 1368 class OldRegionSetChecker : public HeapRegionSetChecker { 1369 public: 1370 void check_mt_safety() { 1371 // Master Old Set MT safety protocol: 1372 // (a) If we're at a safepoint, operations on the master old set 1373 // should be invoked: 1374 // - by the VM thread (which will serialize them), or 1375 // - by the GC workers while holding the FreeList_lock, if we're 1376 // at a safepoint for an evacuation pause (this lock is taken 1377 // anyway when an GC alloc region is retired so that a new one 1378 // is allocated from the free list), or 1379 // - by the GC workers while holding the OldSets_lock, if we're at a 1380 // safepoint for a cleanup pause. 1381 // (b) If we're not at a safepoint, operations on the master old set 1382 // should be invoked while holding the Heap_lock. 1383 1384 if (SafepointSynchronize::is_at_safepoint()) { 1385 guarantee(Thread::current()->is_VM_thread() || 1386 FreeList_lock->owned_by_self() || OldSets_lock->owned_by_self(), 1387 "master old set MT safety protocol at a safepoint"); 1388 } else { 1389 guarantee(Heap_lock->owned_by_self(), "master old set MT safety protocol outside a safepoint"); 1390 } 1391 } 1392 bool is_correct_type(HeapRegion* hr) { return hr->is_old(); } 1393 const char* get_description() { return "Old Regions"; } 1394 }; 1395 1396 class ArchiveRegionSetChecker : public HeapRegionSetChecker { 1397 public: 1398 void check_mt_safety() { 1399 guarantee(!Universe::is_fully_initialized() || SafepointSynchronize::is_at_safepoint(), 1400 "May only change archive regions during initialization or safepoint."); 1401 } 1402 bool is_correct_type(HeapRegion* hr) { return hr->is_archive(); } 1403 const char* get_description() { return "Archive Regions"; } 1404 }; 1405 1406 class HumongousRegionSetChecker : public HeapRegionSetChecker { 1407 public: 1408 void check_mt_safety() { 1409 // Humongous Set MT safety protocol: 1410 // (a) If we're at a safepoint, operations on the master humongous 1411 // set should be invoked by either the VM thread (which will 1412 // serialize them) or by the GC workers while holding the 1413 // OldSets_lock. 1414 // (b) If we're not at a safepoint, operations on the master 1415 // humongous set should be invoked while holding the Heap_lock. 1416 1417 if (SafepointSynchronize::is_at_safepoint()) { 1418 guarantee(Thread::current()->is_VM_thread() || 1419 OldSets_lock->owned_by_self(), 1420 "master humongous set MT safety protocol at a safepoint"); 1421 } else { 1422 guarantee(Heap_lock->owned_by_self(), 1423 "master humongous set MT safety protocol outside a safepoint"); 1424 } 1425 } 1426 bool is_correct_type(HeapRegion* hr) { return hr->is_humongous(); } 1427 const char* get_description() { return "Humongous Regions"; } 1428 }; 1429 1430 G1CollectedHeap::G1CollectedHeap() : 1431 CollectedHeap(), 1432 _service_thread(NULL), 1433 _periodic_gc_task(NULL), 1434 _free_segmented_array_memory_task(NULL), 1435 _workers(NULL), 1436 _card_table(NULL), 1437 _collection_pause_end(Ticks::now()), 1438 _soft_ref_policy(), 1439 _old_set("Old Region Set", new OldRegionSetChecker()), 1440 _archive_set("Archive Region Set", new ArchiveRegionSetChecker()), 1441 _humongous_set("Humongous Region Set", new HumongousRegionSetChecker()), 1442 _bot(NULL), 1443 _listener(), 1444 _numa(G1NUMA::create()), 1445 _hrm(), 1446 _allocator(NULL), 1447 _evac_failure_injector(), 1448 _verifier(NULL), 1449 _summary_bytes_used(0), 1450 _bytes_used_during_gc(0), 1451 _archive_allocator(nullptr), 1452 _survivor_evac_stats("Young", YoungPLABSize, PLABWeight), 1453 _old_evac_stats("Old", OldPLABSize, PLABWeight), 1454 _monitoring_support(nullptr), 1455 _humongous_reclaim_candidates(), 1456 _num_humongous_objects(0), 1457 _num_humongous_reclaim_candidates(0), 1458 _hr_printer(), 1459 _collector_state(), 1460 _old_marking_cycles_started(0), 1461 _old_marking_cycles_completed(0), 1462 _eden(), 1463 _survivor(), 1464 _gc_timer_stw(new (ResourceObj::C_HEAP, mtGC) STWGCTimer()), 1465 _gc_tracer_stw(new (ResourceObj::C_HEAP, mtGC) G1NewTracer()), 1466 _policy(new G1Policy(_gc_timer_stw)), 1467 _heap_sizing_policy(NULL), 1468 _collection_set(this, _policy), 1469 _hot_card_cache(NULL), 1470 _rem_set(NULL), 1471 _card_set_config(), 1472 _cm(NULL), 1473 _cm_thread(NULL), 1474 _cr(NULL), 1475 _task_queues(NULL), 1476 _ref_processor_stw(NULL), 1477 _is_alive_closure_stw(this), 1478 _is_subject_to_discovery_stw(this), 1479 _ref_processor_cm(NULL), 1480 _is_alive_closure_cm(this), 1481 _is_subject_to_discovery_cm(this), 1482 _region_attr() { 1483 1484 _verifier = new G1HeapVerifier(this); 1485 1486 _allocator = new G1Allocator(this); 1487 1488 _heap_sizing_policy = G1HeapSizingPolicy::create(this, _policy->analytics()); 1489 1490 _humongous_object_threshold_in_words = humongous_threshold_for(HeapRegion::GrainWords); 1491 1492 // Override the default _filler_array_max_size so that no humongous filler 1493 // objects are created. 1494 _filler_array_max_size = _humongous_object_threshold_in_words; 1495 1496 uint n_queues = ParallelGCThreads; 1497 _task_queues = new G1ScannerTasksQueueSet(n_queues); 1498 1499 for (uint i = 0; i < n_queues; i++) { 1500 G1ScannerTasksQueue* q = new G1ScannerTasksQueue(); 1501 _task_queues->register_queue(i, q); 1502 } 1503 1504 _gc_tracer_stw->initialize(); 1505 1506 guarantee(_task_queues != NULL, "task_queues allocation failure."); 1507 } 1508 1509 G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description, 1510 size_t size, 1511 size_t translation_factor) { 1512 size_t preferred_page_size = os::page_size_for_region_unaligned(size, 1); 1513 // Allocate a new reserved space, preferring to use large pages. 1514 ReservedSpace rs(size, preferred_page_size); 1515 size_t page_size = rs.page_size(); 1516 G1RegionToSpaceMapper* result = 1517 G1RegionToSpaceMapper::create_mapper(rs, 1518 size, 1519 page_size, 1520 HeapRegion::GrainBytes, 1521 translation_factor, 1522 mtGC); 1523 1524 os::trace_page_sizes_for_requested_size(description, 1525 size, 1526 page_size, 1527 preferred_page_size, 1528 rs.base(), 1529 rs.size()); 1530 1531 return result; 1532 } 1533 1534 jint G1CollectedHeap::initialize_concurrent_refinement() { 1535 jint ecode = JNI_OK; 1536 _cr = G1ConcurrentRefine::create(&ecode); 1537 return ecode; 1538 } 1539 1540 jint G1CollectedHeap::initialize_service_thread() { 1541 _service_thread = new G1ServiceThread(); 1542 if (_service_thread->osthread() == NULL) { 1543 vm_shutdown_during_initialization("Could not create G1ServiceThread"); 1544 return JNI_ENOMEM; 1545 } 1546 return JNI_OK; 1547 } 1548 1549 jint G1CollectedHeap::initialize() { 1550 1551 // Necessary to satisfy locking discipline assertions. 1552 1553 MutexLocker x(Heap_lock); 1554 1555 // While there are no constraints in the GC code that HeapWordSize 1556 // be any particular value, there are multiple other areas in the 1557 // system which believe this to be true (e.g. oop->object_size in some 1558 // cases incorrectly returns the size in wordSize units rather than 1559 // HeapWordSize). 1560 guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); 1561 1562 size_t init_byte_size = InitialHeapSize; 1563 size_t reserved_byte_size = G1Arguments::heap_reserved_size_bytes(); 1564 1565 // Ensure that the sizes are properly aligned. 1566 Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); 1567 Universe::check_alignment(reserved_byte_size, HeapRegion::GrainBytes, "g1 heap"); 1568 Universe::check_alignment(reserved_byte_size, HeapAlignment, "g1 heap"); 1569 1570 // Reserve the maximum. 1571 1572 // When compressed oops are enabled, the preferred heap base 1573 // is calculated by subtracting the requested size from the 1574 // 32Gb boundary and using the result as the base address for 1575 // heap reservation. If the requested size is not aligned to 1576 // HeapRegion::GrainBytes (i.e. the alignment that is passed 1577 // into the ReservedHeapSpace constructor) then the actual 1578 // base of the reserved heap may end up differing from the 1579 // address that was requested (i.e. the preferred heap base). 1580 // If this happens then we could end up using a non-optimal 1581 // compressed oops mode. 1582 1583 ReservedHeapSpace heap_rs = Universe::reserve_heap(reserved_byte_size, 1584 HeapAlignment); 1585 1586 initialize_reserved_region(heap_rs); 1587 1588 // Create the barrier set for the entire reserved region. 1589 G1CardTable* ct = new G1CardTable(heap_rs.region()); 1590 ct->initialize(); 1591 G1BarrierSet* bs = new G1BarrierSet(ct); 1592 bs->initialize(); 1593 assert(bs->is_a(BarrierSet::G1BarrierSet), "sanity"); 1594 BarrierSet::set_barrier_set(bs); 1595 _card_table = ct; 1596 1597 { 1598 G1SATBMarkQueueSet& satbqs = bs->satb_mark_queue_set(); 1599 satbqs.set_process_completed_buffers_threshold(G1SATBProcessCompletedThreshold); 1600 satbqs.set_buffer_enqueue_threshold_percentage(G1SATBBufferEnqueueingThresholdPercent); 1601 } 1602 1603 // Create the hot card cache. 1604 _hot_card_cache = new G1HotCardCache(this); 1605 1606 // Create space mappers. 1607 size_t page_size = heap_rs.page_size(); 1608 G1RegionToSpaceMapper* heap_storage = 1609 G1RegionToSpaceMapper::create_mapper(heap_rs, 1610 heap_rs.size(), 1611 page_size, 1612 HeapRegion::GrainBytes, 1613 1, 1614 mtJavaHeap); 1615 if(heap_storage == NULL) { 1616 vm_shutdown_during_initialization("Could not initialize G1 heap"); 1617 return JNI_ERR; 1618 } 1619 1620 os::trace_page_sizes("Heap", 1621 MinHeapSize, 1622 reserved_byte_size, 1623 page_size, 1624 heap_rs.base(), 1625 heap_rs.size()); 1626 heap_storage->set_mapping_changed_listener(&_listener); 1627 1628 // Create storage for the BOT, card table, card counts table (hot card cache) and the bitmaps. 1629 G1RegionToSpaceMapper* bot_storage = 1630 create_aux_memory_mapper("Block Offset Table", 1631 G1BlockOffsetTable::compute_size(heap_rs.size() / HeapWordSize), 1632 G1BlockOffsetTable::heap_map_factor()); 1633 1634 G1RegionToSpaceMapper* cardtable_storage = 1635 create_aux_memory_mapper("Card Table", 1636 G1CardTable::compute_size(heap_rs.size() / HeapWordSize), 1637 G1CardTable::heap_map_factor()); 1638 1639 G1RegionToSpaceMapper* card_counts_storage = 1640 create_aux_memory_mapper("Card Counts Table", 1641 G1CardCounts::compute_size(heap_rs.size() / HeapWordSize), 1642 G1CardCounts::heap_map_factor()); 1643 1644 size_t bitmap_size = G1CMBitMap::compute_size(heap_rs.size()); 1645 G1RegionToSpaceMapper* prev_bitmap_storage = 1646 create_aux_memory_mapper("Prev Bitmap", bitmap_size, G1CMBitMap::heap_map_factor()); 1647 G1RegionToSpaceMapper* next_bitmap_storage = 1648 create_aux_memory_mapper("Next Bitmap", bitmap_size, G1CMBitMap::heap_map_factor()); 1649 1650 _hrm.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage); 1651 _card_table->initialize(cardtable_storage); 1652 1653 // Do later initialization work for concurrent refinement. 1654 _hot_card_cache->initialize(card_counts_storage); 1655 1656 // 6843694 - ensure that the maximum region index can fit 1657 // in the remembered set structures. 1658 const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1; 1659 guarantee((max_reserved_regions() - 1) <= max_region_idx, "too many regions"); 1660 1661 // The G1FromCardCache reserves card with value 0 as "invalid", so the heap must not 1662 // start within the first card. 1663 guarantee((uintptr_t)(heap_rs.base()) >= G1CardTable::card_size(), "Java heap must not start within the first card."); 1664 G1FromCardCache::initialize(max_reserved_regions()); 1665 // Also create a G1 rem set. 1666 _rem_set = new G1RemSet(this, _card_table, _hot_card_cache); 1667 _rem_set->initialize(max_reserved_regions()); 1668 1669 size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1; 1670 guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized"); 1671 guarantee(HeapRegion::CardsPerRegion < max_cards_per_region, 1672 "too many cards per region"); 1673 1674 HeapRegionRemSet::initialize(_reserved); 1675 1676 FreeRegionList::set_unrealistically_long_length(max_regions() + 1); 1677 1678 _bot = new G1BlockOffsetTable(reserved(), bot_storage); 1679 1680 { 1681 size_t granularity = HeapRegion::GrainBytes; 1682 1683 _region_attr.initialize(reserved(), granularity); 1684 _humongous_reclaim_candidates.initialize(reserved(), granularity); 1685 } 1686 1687 _workers = new WorkerThreads("GC Thread", ParallelGCThreads); 1688 if (_workers == NULL) { 1689 return JNI_ENOMEM; 1690 } 1691 _workers->initialize_workers(); 1692 1693 _numa->set_region_info(HeapRegion::GrainBytes, page_size); 1694 1695 // Create the G1ConcurrentMark data structure and thread. 1696 // (Must do this late, so that "max_[reserved_]regions" is defined.) 1697 _cm = new G1ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage); 1698 _cm_thread = _cm->cm_thread(); 1699 1700 // Now expand into the initial heap size. 1701 if (!expand(init_byte_size, _workers)) { 1702 vm_shutdown_during_initialization("Failed to allocate initial heap."); 1703 return JNI_ENOMEM; 1704 } 1705 1706 // Perform any initialization actions delegated to the policy. 1707 policy()->init(this, &_collection_set); 1708 1709 jint ecode = initialize_concurrent_refinement(); 1710 if (ecode != JNI_OK) { 1711 return ecode; 1712 } 1713 1714 ecode = initialize_service_thread(); 1715 if (ecode != JNI_OK) { 1716 return ecode; 1717 } 1718 1719 // Initialize and schedule sampling task on service thread. 1720 _rem_set->initialize_sampling_task(service_thread()); 1721 1722 // Create and schedule the periodic gc task on the service thread. 1723 _periodic_gc_task = new G1PeriodicGCTask("Periodic GC Task"); 1724 _service_thread->register_task(_periodic_gc_task); 1725 1726 _free_segmented_array_memory_task = new G1SegmentedArrayFreeMemoryTask("Card Set Free Memory Task"); 1727 _service_thread->register_task(_free_segmented_array_memory_task); 1728 1729 // Here we allocate the dummy HeapRegion that is required by the 1730 // G1AllocRegion class. 1731 HeapRegion* dummy_region = _hrm.get_dummy_region(); 1732 1733 // We'll re-use the same region whether the alloc region will 1734 // require BOT updates or not and, if it doesn't, then a non-young 1735 // region will complain that it cannot support allocations without 1736 // BOT updates. So we'll tag the dummy region as eden to avoid that. 1737 dummy_region->set_eden(); 1738 // Make sure it's full. 1739 dummy_region->set_top(dummy_region->end()); 1740 G1AllocRegion::setup(this, dummy_region); 1741 1742 _allocator->init_mutator_alloc_regions(); 1743 1744 // Do create of the monitoring and management support so that 1745 // values in the heap have been properly initialized. 1746 _monitoring_support = new G1MonitoringSupport(this); 1747 1748 _collection_set.initialize(max_reserved_regions()); 1749 1750 evac_failure_injector()->reset(); 1751 1752 G1InitLogger::print(); 1753 1754 return JNI_OK; 1755 } 1756 1757 bool G1CollectedHeap::concurrent_mark_is_terminating() const { 1758 return _cm_thread->should_terminate(); 1759 } 1760 1761 void G1CollectedHeap::stop() { 1762 // Stop all concurrent threads. We do this to make sure these threads 1763 // do not continue to execute and access resources (e.g. logging) 1764 // that are destroyed during shutdown. 1765 _cr->stop(); 1766 _service_thread->stop(); 1767 _cm_thread->stop(); 1768 } 1769 1770 void G1CollectedHeap::safepoint_synchronize_begin() { 1771 SuspendibleThreadSet::synchronize(); 1772 } 1773 1774 void G1CollectedHeap::safepoint_synchronize_end() { 1775 SuspendibleThreadSet::desynchronize(); 1776 } 1777 1778 void G1CollectedHeap::post_initialize() { 1779 CollectedHeap::post_initialize(); 1780 ref_processing_init(); 1781 } 1782 1783 void G1CollectedHeap::ref_processing_init() { 1784 // Reference processing in G1 currently works as follows: 1785 // 1786 // * There are two reference processor instances. One is 1787 // used to record and process discovered references 1788 // during concurrent marking; the other is used to 1789 // record and process references during STW pauses 1790 // (both full and incremental). 1791 // * Both ref processors need to 'span' the entire heap as 1792 // the regions in the collection set may be dotted around. 1793 // 1794 // * For the concurrent marking ref processor: 1795 // * Reference discovery is enabled at concurrent start. 1796 // * Reference discovery is disabled and the discovered 1797 // references processed etc during remarking. 1798 // * Reference discovery is MT (see below). 1799 // * Reference discovery requires a barrier (see below). 1800 // * Reference processing may or may not be MT 1801 // (depending on the value of ParallelRefProcEnabled 1802 // and ParallelGCThreads). 1803 // * A full GC disables reference discovery by the CM 1804 // ref processor and abandons any entries on it's 1805 // discovered lists. 1806 // 1807 // * For the STW processor: 1808 // * Non MT discovery is enabled at the start of a full GC. 1809 // * Processing and enqueueing during a full GC is non-MT. 1810 // * During a full GC, references are processed after marking. 1811 // 1812 // * Discovery (may or may not be MT) is enabled at the start 1813 // of an incremental evacuation pause. 1814 // * References are processed near the end of a STW evacuation pause. 1815 // * For both types of GC: 1816 // * Discovery is atomic - i.e. not concurrent. 1817 // * Reference discovery will not need a barrier. 1818 1819 // Concurrent Mark ref processor 1820 _ref_processor_cm = 1821 new ReferenceProcessor(&_is_subject_to_discovery_cm, 1822 ParallelGCThreads, // degree of mt processing 1823 // We discover with the gc worker threads during Remark, so both 1824 // thread counts must be considered for discovery. 1825 MAX2(ParallelGCThreads, ConcGCThreads), // degree of mt discovery 1826 true, // Reference discovery is concurrent 1827 &_is_alive_closure_cm); // is alive closure 1828 1829 // STW ref processor 1830 _ref_processor_stw = 1831 new ReferenceProcessor(&_is_subject_to_discovery_stw, 1832 ParallelGCThreads, // degree of mt processing 1833 ParallelGCThreads, // degree of mt discovery 1834 false, // Reference discovery is not concurrent 1835 &_is_alive_closure_stw); // is alive closure 1836 } 1837 1838 SoftRefPolicy* G1CollectedHeap::soft_ref_policy() { 1839 return &_soft_ref_policy; 1840 } 1841 1842 size_t G1CollectedHeap::capacity() const { 1843 return _hrm.length() * HeapRegion::GrainBytes; 1844 } 1845 1846 size_t G1CollectedHeap::unused_committed_regions_in_bytes() const { 1847 return _hrm.total_free_bytes(); 1848 } 1849 1850 void G1CollectedHeap::iterate_hcc_closure(G1CardTableEntryClosure* cl, uint worker_id) { 1851 _hot_card_cache->drain(cl, worker_id); 1852 } 1853 1854 // Computes the sum of the storage used by the various regions. 1855 size_t G1CollectedHeap::used() const { 1856 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions(); 1857 assert(_archive_allocator == nullptr, "must be, should not contribute to used"); 1858 return result; 1859 } 1860 1861 size_t G1CollectedHeap::used_unlocked() const { 1862 return _summary_bytes_used; 1863 } 1864 1865 class SumUsedClosure: public HeapRegionClosure { 1866 size_t _used; 1867 public: 1868 SumUsedClosure() : _used(0) {} 1869 bool do_heap_region(HeapRegion* r) { 1870 _used += r->used(); 1871 return false; 1872 } 1873 size_t result() { return _used; } 1874 }; 1875 1876 size_t G1CollectedHeap::recalculate_used() const { 1877 SumUsedClosure blk; 1878 heap_region_iterate(&blk); 1879 return blk.result(); 1880 } 1881 1882 bool G1CollectedHeap::is_user_requested_concurrent_full_gc(GCCause::Cause cause) { 1883 switch (cause) { 1884 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent; 1885 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent; 1886 case GCCause::_wb_conc_mark: return true; 1887 default : return false; 1888 } 1889 } 1890 1891 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { 1892 switch (cause) { 1893 case GCCause::_g1_humongous_allocation: return true; 1894 case GCCause::_g1_periodic_collection: return G1PeriodicGCInvokesConcurrent; 1895 case GCCause::_wb_breakpoint: return true; 1896 default: return is_user_requested_concurrent_full_gc(cause); 1897 } 1898 } 1899 1900 #ifndef PRODUCT 1901 void G1CollectedHeap::allocate_dummy_regions() { 1902 // Let's fill up most of the region 1903 size_t word_size = HeapRegion::GrainWords - 1024; 1904 // And as a result the region we'll allocate will be humongous. 1905 guarantee(is_humongous(word_size), "sanity"); 1906 1907 // _filler_array_max_size is set to humongous object threshold 1908 // but temporarily change it to use CollectedHeap::fill_with_object(). 1909 AutoModifyRestore<size_t> temporarily(_filler_array_max_size, word_size); 1910 1911 for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) { 1912 // Let's use the existing mechanism for the allocation 1913 HeapWord* dummy_obj = humongous_obj_allocate(word_size); 1914 if (dummy_obj != NULL) { 1915 MemRegion mr(dummy_obj, word_size); 1916 CollectedHeap::fill_with_object(mr); 1917 } else { 1918 // If we can't allocate once, we probably cannot allocate 1919 // again. Let's get out of the loop. 1920 break; 1921 } 1922 } 1923 } 1924 #endif // !PRODUCT 1925 1926 void G1CollectedHeap::increment_old_marking_cycles_started() { 1927 assert(_old_marking_cycles_started == _old_marking_cycles_completed || 1928 _old_marking_cycles_started == _old_marking_cycles_completed + 1, 1929 "Wrong marking cycle count (started: %d, completed: %d)", 1930 _old_marking_cycles_started, _old_marking_cycles_completed); 1931 1932 _old_marking_cycles_started++; 1933 } 1934 1935 void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent, 1936 bool whole_heap_examined) { 1937 MonitorLocker ml(G1OldGCCount_lock, Mutex::_no_safepoint_check_flag); 1938 1939 // We assume that if concurrent == true, then the caller is a 1940 // concurrent thread that was joined the Suspendible Thread 1941 // Set. If there's ever a cheap way to check this, we should add an 1942 // assert here. 1943 1944 // Given that this method is called at the end of a Full GC or of a 1945 // concurrent cycle, and those can be nested (i.e., a Full GC can 1946 // interrupt a concurrent cycle), the number of full collections 1947 // completed should be either one (in the case where there was no 1948 // nesting) or two (when a Full GC interrupted a concurrent cycle) 1949 // behind the number of full collections started. 1950 1951 // This is the case for the inner caller, i.e. a Full GC. 1952 assert(concurrent || 1953 (_old_marking_cycles_started == _old_marking_cycles_completed + 1) || 1954 (_old_marking_cycles_started == _old_marking_cycles_completed + 2), 1955 "for inner caller (Full GC): _old_marking_cycles_started = %u " 1956 "is inconsistent with _old_marking_cycles_completed = %u", 1957 _old_marking_cycles_started, _old_marking_cycles_completed); 1958 1959 // This is the case for the outer caller, i.e. the concurrent cycle. 1960 assert(!concurrent || 1961 (_old_marking_cycles_started == _old_marking_cycles_completed + 1), 1962 "for outer caller (concurrent cycle): " 1963 "_old_marking_cycles_started = %u " 1964 "is inconsistent with _old_marking_cycles_completed = %u", 1965 _old_marking_cycles_started, _old_marking_cycles_completed); 1966 1967 _old_marking_cycles_completed += 1; 1968 if (whole_heap_examined) { 1969 // Signal that we have completed a visit to all live objects. 1970 record_whole_heap_examined_timestamp(); 1971 } 1972 1973 // We need to clear the "in_progress" flag in the CM thread before 1974 // we wake up any waiters (especially when ExplicitInvokesConcurrent 1975 // is set) so that if a waiter requests another System.gc() it doesn't 1976 // incorrectly see that a marking cycle is still in progress. 1977 if (concurrent) { 1978 _cm_thread->set_idle(); 1979 } 1980 1981 // Notify threads waiting in System.gc() (with ExplicitGCInvokesConcurrent) 1982 // for a full GC to finish that their wait is over. 1983 ml.notify_all(); 1984 } 1985 1986 // Helper for collect(). 1987 static G1GCCounters collection_counters(G1CollectedHeap* g1h) { 1988 MutexLocker ml(Heap_lock); 1989 return G1GCCounters(g1h); 1990 } 1991 1992 void G1CollectedHeap::collect(GCCause::Cause cause) { 1993 try_collect(cause, collection_counters(this)); 1994 } 1995 1996 // Return true if (x < y) with allowance for wraparound. 1997 static bool gc_counter_less_than(uint x, uint y) { 1998 return (x - y) > (UINT_MAX/2); 1999 } 2000 2001 // LOG_COLLECT_CONCURRENTLY(cause, msg, args...) 2002 // Macro so msg printing is format-checked. 2003 #define LOG_COLLECT_CONCURRENTLY(cause, ...) \ 2004 do { \ 2005 LogTarget(Trace, gc) LOG_COLLECT_CONCURRENTLY_lt; \ 2006 if (LOG_COLLECT_CONCURRENTLY_lt.is_enabled()) { \ 2007 ResourceMark rm; /* For thread name. */ \ 2008 LogStream LOG_COLLECT_CONCURRENTLY_s(&LOG_COLLECT_CONCURRENTLY_lt); \ 2009 LOG_COLLECT_CONCURRENTLY_s.print("%s: Try Collect Concurrently (%s): ", \ 2010 Thread::current()->name(), \ 2011 GCCause::to_string(cause)); \ 2012 LOG_COLLECT_CONCURRENTLY_s.print(__VA_ARGS__); \ 2013 } \ 2014 } while (0) 2015 2016 #define LOG_COLLECT_CONCURRENTLY_COMPLETE(cause, result) \ 2017 LOG_COLLECT_CONCURRENTLY(cause, "complete %s", BOOL_TO_STR(result)) 2018 2019 bool G1CollectedHeap::try_collect_concurrently(GCCause::Cause cause, 2020 uint gc_counter, 2021 uint old_marking_started_before) { 2022 assert_heap_not_locked(); 2023 assert(should_do_concurrent_full_gc(cause), 2024 "Non-concurrent cause %s", GCCause::to_string(cause)); 2025 2026 for (uint i = 1; true; ++i) { 2027 // Try to schedule concurrent start evacuation pause that will 2028 // start a concurrent cycle. 2029 LOG_COLLECT_CONCURRENTLY(cause, "attempt %u", i); 2030 VM_G1TryInitiateConcMark op(gc_counter, 2031 cause, 2032 policy()->max_pause_time_ms()); 2033 VMThread::execute(&op); 2034 2035 // Request is trivially finished. 2036 if (cause == GCCause::_g1_periodic_collection) { 2037 LOG_COLLECT_CONCURRENTLY_COMPLETE(cause, op.gc_succeeded()); 2038 return op.gc_succeeded(); 2039 } 2040 2041 // If VMOp skipped initiating concurrent marking cycle because 2042 // we're terminating, then we're done. 2043 if (op.terminating()) { 2044 LOG_COLLECT_CONCURRENTLY(cause, "skipped: terminating"); 2045 return false; 2046 } 2047 2048 // Lock to get consistent set of values. 2049 uint old_marking_started_after; 2050 uint old_marking_completed_after; 2051 { 2052 MutexLocker ml(Heap_lock); 2053 // Update gc_counter for retrying VMOp if needed. Captured here to be 2054 // consistent with the values we use below for termination tests. If 2055 // a retry is needed after a possible wait, and another collection 2056 // occurs in the meantime, it will cause our retry to be skipped and 2057 // we'll recheck for termination with updated conditions from that 2058 // more recent collection. That's what we want, rather than having 2059 // our retry possibly perform an unnecessary collection. 2060 gc_counter = total_collections(); 2061 old_marking_started_after = _old_marking_cycles_started; 2062 old_marking_completed_after = _old_marking_cycles_completed; 2063 } 2064 2065 if (cause == GCCause::_wb_breakpoint) { 2066 if (op.gc_succeeded()) { 2067 LOG_COLLECT_CONCURRENTLY_COMPLETE(cause, true); 2068 return true; 2069 } 2070 // When _wb_breakpoint there can't be another cycle or deferred. 2071 assert(!op.cycle_already_in_progress(), "invariant"); 2072 assert(!op.whitebox_attached(), "invariant"); 2073 // Concurrent cycle attempt might have been cancelled by some other 2074 // collection, so retry. Unlike other cases below, we want to retry 2075 // even if cancelled by a STW full collection, because we really want 2076 // to start a concurrent cycle. 2077 if (old_marking_started_before != old_marking_started_after) { 2078 LOG_COLLECT_CONCURRENTLY(cause, "ignoring STW full GC"); 2079 old_marking_started_before = old_marking_started_after; 2080 } 2081 } else if (!GCCause::is_user_requested_gc(cause)) { 2082 // For an "automatic" (not user-requested) collection, we just need to 2083 // ensure that progress is made. 2084 // 2085 // Request is finished if any of 2086 // (1) the VMOp successfully performed a GC, 2087 // (2) a concurrent cycle was already in progress, 2088 // (3) whitebox is controlling concurrent cycles, 2089 // (4) a new cycle was started (by this thread or some other), or 2090 // (5) a Full GC was performed. 2091 // Cases (4) and (5) are detected together by a change to 2092 // _old_marking_cycles_started. 2093 // 2094 // Note that (1) does not imply (4). If we're still in the mixed 2095 // phase of an earlier concurrent collection, the request to make the 2096 // collection a concurrent start won't be honored. If we don't check for 2097 // both conditions we'll spin doing back-to-back collections. 2098 if (op.gc_succeeded() || 2099 op.cycle_already_in_progress() || 2100 op.whitebox_attached() || 2101 (old_marking_started_before != old_marking_started_after)) { 2102 LOG_COLLECT_CONCURRENTLY_COMPLETE(cause, true); 2103 return true; 2104 } 2105 } else { // User-requested GC. 2106 // For a user-requested collection, we want to ensure that a complete 2107 // full collection has been performed before returning, but without 2108 // waiting for more than needed. 2109 2110 // For user-requested GCs (unlike non-UR), a successful VMOp implies a 2111 // new cycle was started. That's good, because it's not clear what we 2112 // should do otherwise. Trying again just does back to back GCs. 2113 // Can't wait for someone else to start a cycle. And returning fails 2114 // to meet the goal of ensuring a full collection was performed. 2115 assert(!op.gc_succeeded() || 2116 (old_marking_started_before != old_marking_started_after), 2117 "invariant: succeeded %s, started before %u, started after %u", 2118 BOOL_TO_STR(op.gc_succeeded()), 2119 old_marking_started_before, old_marking_started_after); 2120 2121 // Request is finished if a full collection (concurrent or stw) 2122 // was started after this request and has completed, e.g. 2123 // started_before < completed_after. 2124 if (gc_counter_less_than(old_marking_started_before, 2125 old_marking_completed_after)) { 2126 LOG_COLLECT_CONCURRENTLY_COMPLETE(cause, true); 2127 return true; 2128 } 2129 2130 if (old_marking_started_after != old_marking_completed_after) { 2131 // If there is an in-progress cycle (possibly started by us), then 2132 // wait for that cycle to complete, e.g. 2133 // while completed_now < started_after. 2134 LOG_COLLECT_CONCURRENTLY(cause, "wait"); 2135 MonitorLocker ml(G1OldGCCount_lock); 2136 while (gc_counter_less_than(_old_marking_cycles_completed, 2137 old_marking_started_after)) { 2138 ml.wait(); 2139 } 2140 // Request is finished if the collection we just waited for was 2141 // started after this request. 2142 if (old_marking_started_before != old_marking_started_after) { 2143 LOG_COLLECT_CONCURRENTLY(cause, "complete after wait"); 2144 return true; 2145 } 2146 } 2147 2148 // If VMOp was successful then it started a new cycle that the above 2149 // wait &etc should have recognized as finishing this request. This 2150 // differs from a non-user-request, where gc_succeeded does not imply 2151 // a new cycle was started. 2152 assert(!op.gc_succeeded(), "invariant"); 2153 2154 if (op.cycle_already_in_progress()) { 2155 // If VMOp failed because a cycle was already in progress, it 2156 // is now complete. But it didn't finish this user-requested 2157 // GC, so try again. 2158 LOG_COLLECT_CONCURRENTLY(cause, "retry after in-progress"); 2159 continue; 2160 } else if (op.whitebox_attached()) { 2161 // If WhiteBox wants control, wait for notification of a state 2162 // change in the controller, then try again. Don't wait for 2163 // release of control, since collections may complete while in 2164 // control. Note: This won't recognize a STW full collection 2165 // while waiting; we can't wait on multiple monitors. 2166 LOG_COLLECT_CONCURRENTLY(cause, "whitebox control stall"); 2167 MonitorLocker ml(ConcurrentGCBreakpoints::monitor()); 2168 if (ConcurrentGCBreakpoints::is_controlled()) { 2169 ml.wait(); 2170 } 2171 continue; 2172 } 2173 } 2174 2175 // Collection failed and should be retried. 2176 assert(op.transient_failure(), "invariant"); 2177 2178 if (GCLocker::is_active_and_needs_gc()) { 2179 // If GCLocker is active, wait until clear before retrying. 2180 LOG_COLLECT_CONCURRENTLY(cause, "gc-locker stall"); 2181 GCLocker::stall_until_clear(); 2182 } 2183 2184 LOG_COLLECT_CONCURRENTLY(cause, "retry"); 2185 } 2186 } 2187 2188 bool G1CollectedHeap::try_collect(GCCause::Cause cause, 2189 const G1GCCounters& counters_before) { 2190 if (should_do_concurrent_full_gc(cause)) { 2191 return try_collect_concurrently(cause, 2192 counters_before.total_collections(), 2193 counters_before.old_marking_cycles_started()); 2194 } else if (GCLocker::should_discard(cause, counters_before.total_collections())) { 2195 // Indicate failure to be consistent with VMOp failure due to 2196 // another collection slipping in after our gc_count but before 2197 // our request is processed. 2198 return false; 2199 } else if (cause == GCCause::_gc_locker || cause == GCCause::_wb_young_gc 2200 DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) { 2201 2202 // Schedule a standard evacuation pause. We're setting word_size 2203 // to 0 which means that we are not requesting a post-GC allocation. 2204 VM_G1CollectForAllocation op(0, /* word_size */ 2205 counters_before.total_collections(), 2206 cause, 2207 policy()->max_pause_time_ms()); 2208 VMThread::execute(&op); 2209 return op.gc_succeeded(); 2210 } else { 2211 // Schedule a Full GC. 2212 VM_G1CollectFull op(counters_before.total_collections(), 2213 counters_before.total_full_collections(), 2214 cause); 2215 VMThread::execute(&op); 2216 return op.gc_succeeded(); 2217 } 2218 } 2219 2220 void G1CollectedHeap::start_concurrent_gc_for_metadata_allocation(GCCause::Cause gc_cause) { 2221 GCCauseSetter x(this, gc_cause); 2222 2223 // At this point we are supposed to start a concurrent cycle. We 2224 // will do so if one is not already in progress. 2225 bool should_start = policy()->force_concurrent_start_if_outside_cycle(gc_cause); 2226 if (should_start) { 2227 double pause_target = policy()->max_pause_time_ms(); 2228 do_collection_pause_at_safepoint(pause_target); 2229 } 2230 } 2231 2232 bool G1CollectedHeap::is_in(const void* p) const { 2233 return is_in_reserved(p) && _hrm.is_available(addr_to_region((HeapWord*)p)); 2234 } 2235 2236 // Iteration functions. 2237 2238 // Iterates an ObjectClosure over all objects within a HeapRegion. 2239 2240 class IterateObjectClosureRegionClosure: public HeapRegionClosure { 2241 ObjectClosure* _cl; 2242 public: 2243 IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} 2244 bool do_heap_region(HeapRegion* r) { 2245 if (!r->is_continues_humongous()) { 2246 r->object_iterate(_cl); 2247 } 2248 return false; 2249 } 2250 }; 2251 2252 void G1CollectedHeap::object_iterate(ObjectClosure* cl) { 2253 IterateObjectClosureRegionClosure blk(cl); 2254 heap_region_iterate(&blk); 2255 } 2256 2257 class G1ParallelObjectIterator : public ParallelObjectIteratorImpl { 2258 private: 2259 G1CollectedHeap* _heap; 2260 HeapRegionClaimer _claimer; 2261 2262 public: 2263 G1ParallelObjectIterator(uint thread_num) : 2264 _heap(G1CollectedHeap::heap()), 2265 _claimer(thread_num == 0 ? G1CollectedHeap::heap()->workers()->active_workers() : thread_num) {} 2266 2267 virtual void object_iterate(ObjectClosure* cl, uint worker_id) { 2268 _heap->object_iterate_parallel(cl, worker_id, &_claimer); 2269 } 2270 }; 2271 2272 ParallelObjectIteratorImpl* G1CollectedHeap::parallel_object_iterator(uint thread_num) { 2273 return new G1ParallelObjectIterator(thread_num); 2274 } 2275 2276 void G1CollectedHeap::object_iterate_parallel(ObjectClosure* cl, uint worker_id, HeapRegionClaimer* claimer) { 2277 IterateObjectClosureRegionClosure blk(cl); 2278 heap_region_par_iterate_from_worker_offset(&blk, claimer, worker_id); 2279 } 2280 2281 void G1CollectedHeap::keep_alive(oop obj) { 2282 G1BarrierSet::enqueue_preloaded(obj); 2283 } 2284 2285 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const { 2286 _hrm.iterate(cl); 2287 } 2288 2289 void G1CollectedHeap::heap_region_par_iterate_from_worker_offset(HeapRegionClosure* cl, 2290 HeapRegionClaimer *hrclaimer, 2291 uint worker_id) const { 2292 _hrm.par_iterate(cl, hrclaimer, hrclaimer->offset_for_worker(worker_id)); 2293 } 2294 2295 void G1CollectedHeap::heap_region_par_iterate_from_start(HeapRegionClosure* cl, 2296 HeapRegionClaimer *hrclaimer) const { 2297 _hrm.par_iterate(cl, hrclaimer, 0); 2298 } 2299 2300 void G1CollectedHeap::collection_set_iterate_all(HeapRegionClosure* cl) { 2301 _collection_set.iterate(cl); 2302 } 2303 2304 void G1CollectedHeap::collection_set_par_iterate_all(HeapRegionClosure* cl, 2305 HeapRegionClaimer* hr_claimer, 2306 uint worker_id) { 2307 _collection_set.par_iterate(cl, hr_claimer, worker_id); 2308 } 2309 2310 void G1CollectedHeap::collection_set_iterate_increment_from(HeapRegionClosure *cl, 2311 HeapRegionClaimer* hr_claimer, 2312 uint worker_id) { 2313 _collection_set.iterate_incremental_part_from(cl, hr_claimer, worker_id); 2314 } 2315 2316 void G1CollectedHeap::par_iterate_regions_array(HeapRegionClosure* cl, 2317 HeapRegionClaimer* hr_claimer, 2318 const uint regions[], 2319 size_t length, 2320 uint worker_id) const { 2321 assert_at_safepoint(); 2322 if (length == 0) { 2323 return; 2324 } 2325 uint total_workers = workers()->active_workers(); 2326 2327 size_t start_pos = (worker_id * length) / total_workers; 2328 size_t cur_pos = start_pos; 2329 2330 do { 2331 uint region_idx = regions[cur_pos]; 2332 if (hr_claimer == NULL || hr_claimer->claim_region(region_idx)) { 2333 HeapRegion* r = region_at(region_idx); 2334 bool result = cl->do_heap_region(r); 2335 guarantee(!result, "Must not cancel iteration"); 2336 } 2337 2338 cur_pos++; 2339 if (cur_pos == length) { 2340 cur_pos = 0; 2341 } 2342 } while (cur_pos != start_pos); 2343 } 2344 2345 HeapWord* G1CollectedHeap::block_start(const void* addr) const { 2346 HeapRegion* hr = heap_region_containing(addr); 2347 return hr->block_start(addr); 2348 } 2349 2350 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const { 2351 HeapRegion* hr = heap_region_containing(addr); 2352 return hr->block_is_obj(addr); 2353 } 2354 2355 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const { 2356 return (_policy->young_list_target_length() - _survivor.length()) * HeapRegion::GrainBytes; 2357 } 2358 2359 size_t G1CollectedHeap::tlab_used(Thread* ignored) const { 2360 return _eden.length() * HeapRegion::GrainBytes; 2361 } 2362 2363 // For G1 TLABs should not contain humongous objects, so the maximum TLAB size 2364 // must be equal to the humongous object limit. 2365 size_t G1CollectedHeap::max_tlab_size() const { 2366 return align_down(_humongous_object_threshold_in_words, MinObjAlignment); 2367 } 2368 2369 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { 2370 return _allocator->unsafe_max_tlab_alloc(); 2371 } 2372 2373 size_t G1CollectedHeap::max_capacity() const { 2374 return max_regions() * HeapRegion::GrainBytes; 2375 } 2376 2377 void G1CollectedHeap::prepare_for_verify() { 2378 _verifier->prepare_for_verify(); 2379 } 2380 2381 void G1CollectedHeap::verify(VerifyOption vo) { 2382 _verifier->verify(vo); 2383 } 2384 2385 bool G1CollectedHeap::supports_concurrent_gc_breakpoints() const { 2386 return true; 2387 } 2388 2389 bool G1CollectedHeap::is_archived_object(oop object) const { 2390 return object != NULL && heap_region_containing(object)->is_archive(); 2391 } 2392 2393 class PrintRegionClosure: public HeapRegionClosure { 2394 outputStream* _st; 2395 public: 2396 PrintRegionClosure(outputStream* st) : _st(st) {} 2397 bool do_heap_region(HeapRegion* r) { 2398 r->print_on(_st); 2399 return false; 2400 } 2401 }; 2402 2403 bool G1CollectedHeap::is_obj_dead_cond(const oop obj, 2404 const HeapRegion* hr, 2405 const VerifyOption vo) const { 2406 switch (vo) { 2407 case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr); 2408 case VerifyOption_G1UseFullMarking: return is_obj_dead_full(obj, hr); 2409 default: ShouldNotReachHere(); 2410 } 2411 return false; // keep some compilers happy 2412 } 2413 2414 bool G1CollectedHeap::is_obj_dead_cond(const oop obj, 2415 const VerifyOption vo) const { 2416 switch (vo) { 2417 case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj); 2418 case VerifyOption_G1UseFullMarking: return is_obj_dead_full(obj); 2419 default: ShouldNotReachHere(); 2420 } 2421 return false; // keep some compilers happy 2422 } 2423 2424 void G1CollectedHeap::print_heap_regions() const { 2425 LogTarget(Trace, gc, heap, region) lt; 2426 if (lt.is_enabled()) { 2427 LogStream ls(lt); 2428 print_regions_on(&ls); 2429 } 2430 } 2431 2432 void G1CollectedHeap::print_on(outputStream* st) const { 2433 size_t heap_used = Heap_lock->owned_by_self() ? used() : used_unlocked(); 2434 st->print(" %-20s", "garbage-first heap"); 2435 st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", 2436 capacity()/K, heap_used/K); 2437 st->print(" [" PTR_FORMAT ", " PTR_FORMAT ")", 2438 p2i(_hrm.reserved().start()), 2439 p2i(_hrm.reserved().end())); 2440 st->cr(); 2441 st->print(" region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K); 2442 uint young_regions = young_regions_count(); 2443 st->print("%u young (" SIZE_FORMAT "K), ", young_regions, 2444 (size_t) young_regions * HeapRegion::GrainBytes / K); 2445 uint survivor_regions = survivor_regions_count(); 2446 st->print("%u survivors (" SIZE_FORMAT "K)", survivor_regions, 2447 (size_t) survivor_regions * HeapRegion::GrainBytes / K); 2448 st->cr(); 2449 if (_numa->is_enabled()) { 2450 uint num_nodes = _numa->num_active_nodes(); 2451 st->print(" remaining free region(s) on each NUMA node: "); 2452 const int* node_ids = _numa->node_ids(); 2453 for (uint node_index = 0; node_index < num_nodes; node_index++) { 2454 uint num_free_regions = _hrm.num_free_regions(node_index); 2455 st->print("%d=%u ", node_ids[node_index], num_free_regions); 2456 } 2457 st->cr(); 2458 } 2459 MetaspaceUtils::print_on(st); 2460 } 2461 2462 void G1CollectedHeap::print_regions_on(outputStream* st) const { 2463 st->print_cr("Heap Regions: E=young(eden), S=young(survivor), O=old, " 2464 "HS=humongous(starts), HC=humongous(continues), " 2465 "CS=collection set, F=free, " 2466 "OA=open archive, CA=closed archive, " 2467 "TAMS=top-at-mark-start (previous, next)"); 2468 PrintRegionClosure blk(st); 2469 heap_region_iterate(&blk); 2470 } 2471 2472 void G1CollectedHeap::print_extended_on(outputStream* st) const { 2473 print_on(st); 2474 2475 // Print the per-region information. 2476 st->cr(); 2477 print_regions_on(st); 2478 } 2479 2480 void G1CollectedHeap::print_on_error(outputStream* st) const { 2481 this->CollectedHeap::print_on_error(st); 2482 2483 if (_cm != NULL) { 2484 st->cr(); 2485 _cm->print_on_error(st); 2486 } 2487 } 2488 2489 void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const { 2490 workers()->threads_do(tc); 2491 tc->do_thread(_cm_thread); 2492 _cm->threads_do(tc); 2493 _cr->threads_do(tc); 2494 tc->do_thread(_service_thread); 2495 } 2496 2497 void G1CollectedHeap::print_tracing_info() const { 2498 rem_set()->print_summary_info(); 2499 concurrent_mark()->print_summary_info(); 2500 } 2501 2502 #ifndef PRODUCT 2503 // Helpful for debugging RSet issues. 2504 2505 class PrintRSetsClosure : public HeapRegionClosure { 2506 private: 2507 const char* _msg; 2508 size_t _occupied_sum; 2509 2510 public: 2511 bool do_heap_region(HeapRegion* r) { 2512 HeapRegionRemSet* hrrs = r->rem_set(); 2513 size_t occupied = hrrs->occupied(); 2514 _occupied_sum += occupied; 2515 2516 tty->print_cr("Printing RSet for region " HR_FORMAT, HR_FORMAT_PARAMS(r)); 2517 if (occupied == 0) { 2518 tty->print_cr(" RSet is empty"); 2519 } else { 2520 hrrs->print(); 2521 } 2522 tty->print_cr("----------"); 2523 return false; 2524 } 2525 2526 PrintRSetsClosure(const char* msg) : _msg(msg), _occupied_sum(0) { 2527 tty->cr(); 2528 tty->print_cr("========================================"); 2529 tty->print_cr("%s", msg); 2530 tty->cr(); 2531 } 2532 2533 ~PrintRSetsClosure() { 2534 tty->print_cr("Occupied Sum: " SIZE_FORMAT, _occupied_sum); 2535 tty->print_cr("========================================"); 2536 tty->cr(); 2537 } 2538 }; 2539 2540 void G1CollectedHeap::print_cset_rsets() { 2541 PrintRSetsClosure cl("Printing CSet RSets"); 2542 collection_set_iterate_all(&cl); 2543 } 2544 2545 void G1CollectedHeap::print_all_rsets() { 2546 PrintRSetsClosure cl("Printing All RSets");; 2547 heap_region_iterate(&cl); 2548 } 2549 #endif // PRODUCT 2550 2551 bool G1CollectedHeap::print_location(outputStream* st, void* addr) const { 2552 return BlockLocationPrinter<G1CollectedHeap>::print_location(st, addr); 2553 } 2554 2555 G1HeapSummary G1CollectedHeap::create_g1_heap_summary() { 2556 2557 size_t eden_used_bytes = _eden.used_bytes(); 2558 size_t survivor_used_bytes = _survivor.used_bytes(); 2559 size_t heap_used = Heap_lock->owned_by_self() ? used() : used_unlocked(); 2560 2561 size_t eden_capacity_bytes = 2562 (policy()->young_list_target_length() * HeapRegion::GrainBytes) - survivor_used_bytes; 2563 2564 VirtualSpaceSummary heap_summary = create_heap_space_summary(); 2565 return G1HeapSummary(heap_summary, heap_used, eden_used_bytes, 2566 eden_capacity_bytes, survivor_used_bytes, num_regions()); 2567 } 2568 2569 G1EvacSummary G1CollectedHeap::create_g1_evac_summary(G1EvacStats* stats) { 2570 return G1EvacSummary(stats->allocated(), stats->wasted(), stats->undo_wasted(), 2571 stats->unused(), stats->used(), stats->region_end_waste(), 2572 stats->regions_filled(), stats->direct_allocated(), 2573 stats->failure_used(), stats->failure_waste()); 2574 } 2575 2576 void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) { 2577 const G1HeapSummary& heap_summary = create_g1_heap_summary(); 2578 gc_tracer->report_gc_heap_summary(when, heap_summary); 2579 2580 const MetaspaceSummary& metaspace_summary = create_metaspace_summary(); 2581 gc_tracer->report_metaspace_summary(when, metaspace_summary); 2582 } 2583 2584 void G1CollectedHeap::gc_prologue(bool full) { 2585 assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); 2586 2587 // Update common counters. 2588 increment_total_collections(full /* full gc */); 2589 if (full || collector_state()->in_concurrent_start_gc()) { 2590 increment_old_marking_cycles_started(); 2591 } 2592 } 2593 2594 void G1CollectedHeap::gc_epilogue(bool full) { 2595 // Update common counters. 2596 if (full) { 2597 // Update the number of full collections that have been completed. 2598 increment_old_marking_cycles_completed(false /* concurrent */, true /* liveness_completed */); 2599 } 2600 2601 #if COMPILER2_OR_JVMCI 2602 assert(DerivedPointerTable::is_empty(), "derived pointer present"); 2603 #endif 2604 2605 // We have just completed a GC. Update the soft reference 2606 // policy with the new heap occupancy 2607 Universe::heap()->update_capacity_and_used_at_gc(); 2608 2609 _collection_pause_end = Ticks::now(); 2610 2611 _free_segmented_array_memory_task->notify_new_stats(&_young_gen_card_set_stats, 2612 &_collection_set_candidates_card_set_stats); 2613 } 2614 2615 uint G1CollectedHeap::uncommit_regions(uint region_limit) { 2616 return _hrm.uncommit_inactive_regions(region_limit); 2617 } 2618 2619 bool G1CollectedHeap::has_uncommittable_regions() { 2620 return _hrm.has_inactive_regions(); 2621 } 2622 2623 void G1CollectedHeap::uncommit_regions_if_necessary() { 2624 if (has_uncommittable_regions()) { 2625 G1UncommitRegionTask::enqueue(); 2626 } 2627 } 2628 2629 void G1CollectedHeap::verify_numa_regions(const char* desc) { 2630 LogTarget(Trace, gc, heap, verify) lt; 2631 2632 if (lt.is_enabled()) { 2633 LogStream ls(lt); 2634 // Iterate all heap regions to print matching between preferred numa id and actual numa id. 2635 G1NodeIndexCheckClosure cl(desc, _numa, &ls); 2636 heap_region_iterate(&cl); 2637 } 2638 } 2639 2640 HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size, 2641 uint gc_count_before, 2642 bool* succeeded, 2643 GCCause::Cause gc_cause) { 2644 assert_heap_not_locked_and_not_at_safepoint(); 2645 VM_G1CollectForAllocation op(word_size, 2646 gc_count_before, 2647 gc_cause, 2648 policy()->max_pause_time_ms()); 2649 VMThread::execute(&op); 2650 2651 HeapWord* result = op.result(); 2652 bool ret_succeeded = op.prologue_succeeded() && op.gc_succeeded(); 2653 assert(result == NULL || ret_succeeded, 2654 "the result should be NULL if the VM did not succeed"); 2655 *succeeded = ret_succeeded; 2656 2657 assert_heap_not_locked(); 2658 return result; 2659 } 2660 2661 void G1CollectedHeap::start_concurrent_cycle(bool concurrent_operation_is_full_mark) { 2662 assert(!_cm_thread->in_progress(), "Can not start concurrent operation while in progress"); 2663 2664 MutexLocker x(CGC_lock, Mutex::_no_safepoint_check_flag); 2665 if (concurrent_operation_is_full_mark) { 2666 _cm->post_concurrent_mark_start(); 2667 _cm_thread->start_full_mark(); 2668 } else { 2669 _cm->post_concurrent_undo_start(); 2670 _cm_thread->start_undo_mark(); 2671 } 2672 CGC_lock->notify(); 2673 } 2674 2675 bool G1CollectedHeap::is_potential_eager_reclaim_candidate(HeapRegion* r) const { 2676 // We don't nominate objects with many remembered set entries, on 2677 // the assumption that such objects are likely still live. 2678 HeapRegionRemSet* rem_set = r->rem_set(); 2679 2680 return G1EagerReclaimHumongousObjectsWithStaleRefs ? 2681 rem_set->occupancy_less_or_equal_than(G1EagerReclaimRemSetThreshold) : 2682 G1EagerReclaimHumongousObjects && rem_set->is_empty(); 2683 } 2684 2685 #ifndef PRODUCT 2686 void G1CollectedHeap::verify_region_attr_remset_is_tracked() { 2687 class VerifyRegionAttrRemSet : public HeapRegionClosure { 2688 public: 2689 virtual bool do_heap_region(HeapRegion* r) { 2690 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 2691 bool const remset_is_tracked = g1h->region_attr(r->bottom()).remset_is_tracked(); 2692 assert(r->rem_set()->is_tracked() == remset_is_tracked, 2693 "Region %u remset tracking status (%s) different to region attribute (%s)", 2694 r->hrm_index(), BOOL_TO_STR(r->rem_set()->is_tracked()), BOOL_TO_STR(remset_is_tracked)); 2695 return false; 2696 } 2697 } cl; 2698 heap_region_iterate(&cl); 2699 } 2700 #endif 2701 2702 class VerifyRegionRemSetClosure : public HeapRegionClosure { 2703 public: 2704 bool do_heap_region(HeapRegion* hr) { 2705 if (!hr->is_archive() && !hr->is_continues_humongous()) { 2706 hr->verify_rem_set(); 2707 } 2708 return false; 2709 } 2710 }; 2711 2712 void G1CollectedHeap::start_new_collection_set() { 2713 double start = os::elapsedTime(); 2714 2715 collection_set()->start_incremental_building(); 2716 2717 clear_region_attr(); 2718 2719 guarantee(_eden.length() == 0, "eden should have been cleared"); 2720 policy()->transfer_survivors_to_cset(survivor()); 2721 2722 // We redo the verification but now wrt to the new CSet which 2723 // has just got initialized after the previous CSet was freed. 2724 _cm->verify_no_collection_set_oops(); 2725 2726 phase_times()->record_start_new_cset_time_ms((os::elapsedTime() - start) * 1000.0); 2727 } 2728 2729 G1HeapVerifier::G1VerifyType G1CollectedHeap::young_collection_verify_type() const { 2730 if (collector_state()->in_concurrent_start_gc()) { 2731 return G1HeapVerifier::G1VerifyConcurrentStart; 2732 } else if (collector_state()->in_young_only_phase()) { 2733 return G1HeapVerifier::G1VerifyYoungNormal; 2734 } else { 2735 return G1HeapVerifier::G1VerifyMixed; 2736 } 2737 } 2738 2739 void G1CollectedHeap::verify_before_young_collection(G1HeapVerifier::G1VerifyType type) { 2740 if (!VerifyBeforeGC) { 2741 return; 2742 } 2743 Ticks start = Ticks::now(); 2744 _verifier->prepare_for_verify(); 2745 _verifier->verify_region_sets_optional(); 2746 _verifier->verify_dirty_young_regions(); 2747 if (VerifyRememberedSets) { 2748 log_info(gc, verify)("[Verifying RemSets before GC]"); 2749 VerifyRegionRemSetClosure v_cl; 2750 heap_region_iterate(&v_cl); 2751 } 2752 _verifier->verify_before_gc(type); 2753 _verifier->check_bitmaps("GC Start"); 2754 verify_numa_regions("GC Start"); 2755 phase_times()->record_verify_before_time_ms((Ticks::now() - start).seconds() * MILLIUNITS); 2756 } 2757 2758 void G1CollectedHeap::verify_after_young_collection(G1HeapVerifier::G1VerifyType type) { 2759 if (!VerifyAfterGC) { 2760 return; 2761 } 2762 Ticks start = Ticks::now(); 2763 if (VerifyRememberedSets) { 2764 log_info(gc, verify)("[Verifying RemSets after GC]"); 2765 VerifyRegionRemSetClosure v_cl; 2766 heap_region_iterate(&v_cl); 2767 } 2768 _verifier->verify_after_gc(type); 2769 _verifier->check_bitmaps("GC End"); 2770 verify_numa_regions("GC End"); 2771 _verifier->verify_region_sets_optional(); 2772 phase_times()->record_verify_after_time_ms((Ticks::now() - start).seconds() * MILLIUNITS); 2773 } 2774 2775 void G1CollectedHeap::expand_heap_after_young_collection(){ 2776 size_t expand_bytes = _heap_sizing_policy->young_collection_expansion_amount(); 2777 if (expand_bytes > 0) { 2778 // No need for an ergo logging here, 2779 // expansion_amount() does this when it returns a value > 0. 2780 double expand_ms = 0.0; 2781 if (!expand(expand_bytes, _workers, &expand_ms)) { 2782 // We failed to expand the heap. Cannot do anything about it. 2783 } 2784 phase_times()->record_expand_heap_time(expand_ms); 2785 } 2786 } 2787 2788 bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) { 2789 assert_at_safepoint_on_vm_thread(); 2790 guarantee(!is_gc_active(), "collection is not reentrant"); 2791 2792 if (GCLocker::check_active_before_gc()) { 2793 return false; 2794 } 2795 2796 do_collection_pause_at_safepoint_helper(target_pause_time_ms); 2797 return true; 2798 } 2799 2800 G1HeapPrinterMark::G1HeapPrinterMark(G1CollectedHeap* g1h) : _g1h(g1h), _heap_transition(g1h) { 2801 // This summary needs to be printed before incrementing total collections. 2802 _g1h->rem_set()->print_periodic_summary_info("Before GC RS summary", _g1h->total_collections()); 2803 _g1h->print_heap_before_gc(); 2804 _g1h->print_heap_regions(); 2805 } 2806 2807 G1HeapPrinterMark::~G1HeapPrinterMark() { 2808 _g1h->policy()->print_age_table(); 2809 _g1h->rem_set()->print_coarsen_stats(); 2810 // We are at the end of the GC. Total collections has already been increased. 2811 _g1h->rem_set()->print_periodic_summary_info("After GC RS summary", _g1h->total_collections() - 1); 2812 2813 _heap_transition.print(); 2814 _g1h->print_heap_regions(); 2815 _g1h->print_heap_after_gc(); 2816 // Print NUMA statistics. 2817 _g1h->numa()->print_statistics(); 2818 } 2819 2820 G1JFRTracerMark::G1JFRTracerMark(STWGCTimer* timer, GCTracer* tracer) : 2821 _timer(timer), _tracer(tracer) { 2822 2823 _timer->register_gc_start(); 2824 _tracer->report_gc_start(G1CollectedHeap::heap()->gc_cause(), _timer->gc_start()); 2825 G1CollectedHeap::heap()->trace_heap_before_gc(_tracer); 2826 } 2827 2828 G1JFRTracerMark::~G1JFRTracerMark() { 2829 G1CollectedHeap::heap()->trace_heap_after_gc(_tracer); 2830 _timer->register_gc_end(); 2831 _tracer->report_gc_end(_timer->gc_end(), _timer->time_partitions()); 2832 } 2833 2834 void G1CollectedHeap::prepare_tlabs_for_mutator() { 2835 Ticks start = Ticks::now(); 2836 2837 _survivor_evac_stats.adjust_desired_plab_sz(); 2838 _old_evac_stats.adjust_desired_plab_sz(); 2839 2840 allocate_dummy_regions(); 2841 2842 _allocator->init_mutator_alloc_regions(); 2843 2844 resize_all_tlabs(); 2845 2846 phase_times()->record_resize_tlab_time_ms((Ticks::now() - start).seconds() * 1000.0); 2847 } 2848 2849 void G1CollectedHeap::retire_tlabs() { 2850 ensure_parsability(true); 2851 } 2852 2853 void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_pause_time_ms) { 2854 ResourceMark rm; 2855 2856 IsGCActiveMark active_gc_mark; 2857 GCIdMark gc_id_mark; 2858 SvcGCMarker sgcm(SvcGCMarker::MINOR); 2859 2860 GCTraceCPUTime tcpu; 2861 2862 _bytes_used_during_gc = 0; 2863 2864 policy()->decide_on_concurrent_start_pause(); 2865 // Record whether this pause may need to trigger a concurrent operation. Later, 2866 // when we signal the G1ConcurrentMarkThread, the collector state has already 2867 // been reset for the next pause. 2868 bool should_start_concurrent_mark_operation = collector_state()->in_concurrent_start_gc(); 2869 2870 // Perform the collection. 2871 G1YoungCollector collector(gc_cause(), target_pause_time_ms); 2872 collector.collect(); 2873 2874 // It should now be safe to tell the concurrent mark thread to start 2875 // without its logging output interfering with the logging output 2876 // that came from the pause. 2877 if (should_start_concurrent_mark_operation) { 2878 // CAUTION: after the start_concurrent_cycle() call below, the concurrent marking 2879 // thread(s) could be running concurrently with us. Make sure that anything 2880 // after this point does not assume that we are the only GC thread running. 2881 // Note: of course, the actual marking work will not start until the safepoint 2882 // itself is released in SuspendibleThreadSet::desynchronize(). 2883 start_concurrent_cycle(collector.concurrent_operation_is_full_mark()); 2884 ConcurrentGCBreakpoints::notify_idle_to_active(); 2885 } 2886 } 2887 2888 void G1CollectedHeap::complete_cleaning(BoolObjectClosure* is_alive, 2889 bool class_unloading_occurred) { 2890 uint num_workers = workers()->active_workers(); 2891 G1ParallelCleaningTask unlink_task(is_alive, num_workers, class_unloading_occurred); 2892 workers()->run_task(&unlink_task); 2893 } 2894 2895 bool G1STWSubjectToDiscoveryClosure::do_object_b(oop obj) { 2896 assert(obj != NULL, "must not be NULL"); 2897 assert(_g1h->is_in_reserved(obj), "Trying to discover obj " PTR_FORMAT " not in heap", p2i(obj)); 2898 // The areas the CM and STW ref processor manage must be disjoint. The is_in_cset() below 2899 // may falsely indicate that this is not the case here: however the collection set only 2900 // contains old regions when concurrent mark is not running. 2901 return _g1h->is_in_cset(obj) || _g1h->heap_region_containing(obj)->is_survivor(); 2902 } 2903 2904 void G1CollectedHeap::make_pending_list_reachable() { 2905 if (collector_state()->in_concurrent_start_gc()) { 2906 oop pll_head = Universe::reference_pending_list(); 2907 if (pll_head != NULL) { 2908 // Any valid worker id is fine here as we are in the VM thread and single-threaded. 2909 _cm->mark_in_next_bitmap(0 /* worker_id */, pll_head); 2910 } 2911 } 2912 } 2913 2914 static bool do_humongous_object_logging() { 2915 return log_is_enabled(Debug, gc, humongous); 2916 } 2917 2918 bool G1CollectedHeap::should_do_eager_reclaim() const { 2919 // As eager reclaim logging also gives information about humongous objects in 2920 // the heap in general, always do the eager reclaim pass even without known 2921 // candidates. 2922 return (G1EagerReclaimHumongousObjects && 2923 (has_humongous_reclaim_candidates() || do_humongous_object_logging())); 2924 } 2925 2926 void G1CollectedHeap::set_humongous_stats(uint num_humongous_total, uint num_humongous_candidates) { 2927 _num_humongous_objects = num_humongous_total; 2928 _num_humongous_reclaim_candidates = num_humongous_candidates; 2929 } 2930 2931 bool G1CollectedHeap::should_sample_collection_set_candidates() const { 2932 G1CollectionSetCandidates* candidates = G1CollectedHeap::heap()->collection_set()->candidates(); 2933 return candidates != NULL && candidates->num_remaining() > 0; 2934 } 2935 2936 void G1CollectedHeap::set_collection_set_candidates_stats(G1SegmentedArrayMemoryStats& stats) { 2937 _collection_set_candidates_card_set_stats = stats; 2938 } 2939 2940 void G1CollectedHeap::set_young_gen_card_set_stats(const G1SegmentedArrayMemoryStats& stats) { 2941 _young_gen_card_set_stats = stats; 2942 } 2943 2944 void G1CollectedHeap::record_obj_copy_mem_stats() { 2945 policy()->old_gen_alloc_tracker()-> 2946 add_allocated_bytes_since_last_gc(_old_evac_stats.allocated() * HeapWordSize); 2947 2948 _gc_tracer_stw->report_evacuation_statistics(create_g1_evac_summary(&_survivor_evac_stats), 2949 create_g1_evac_summary(&_old_evac_stats)); 2950 } 2951 2952 void G1CollectedHeap::clear_prev_bitmap_for_region(HeapRegion* hr) { 2953 MemRegion mr(hr->bottom(), hr->end()); 2954 concurrent_mark()->clear_range_in_prev_bitmap(mr); 2955 } 2956 2957 void G1CollectedHeap::free_region(HeapRegion* hr, FreeRegionList* free_list) { 2958 assert(!hr->is_free(), "the region should not be free"); 2959 assert(!hr->is_empty(), "the region should not be empty"); 2960 assert(_hrm.is_available(hr->hrm_index()), "region should be committed"); 2961 2962 if (G1VerifyBitmaps) { 2963 clear_prev_bitmap_for_region(hr); 2964 } 2965 2966 // Clear the card counts for this region. 2967 // Note: we only need to do this if the region is not young 2968 // (since we don't refine cards in young regions). 2969 if (!hr->is_young()) { 2970 _hot_card_cache->reset_card_counts(hr); 2971 } 2972 2973 // Reset region metadata to allow reuse. 2974 hr->hr_clear(true /* clear_space */); 2975 _policy->remset_tracker()->update_at_free(hr); 2976 2977 if (free_list != NULL) { 2978 free_list->add_ordered(hr); 2979 } 2980 } 2981 2982 void G1CollectedHeap::free_humongous_region(HeapRegion* hr, 2983 FreeRegionList* free_list) { 2984 assert(hr->is_humongous(), "this is only for humongous regions"); 2985 hr->clear_humongous(); 2986 free_region(hr, free_list); 2987 } 2988 2989 void G1CollectedHeap::remove_from_old_gen_sets(const uint old_regions_removed, 2990 const uint archive_regions_removed, 2991 const uint humongous_regions_removed) { 2992 if (old_regions_removed > 0 || archive_regions_removed > 0 || humongous_regions_removed > 0) { 2993 MutexLocker x(OldSets_lock, Mutex::_no_safepoint_check_flag); 2994 _old_set.bulk_remove(old_regions_removed); 2995 _archive_set.bulk_remove(archive_regions_removed); 2996 _humongous_set.bulk_remove(humongous_regions_removed); 2997 } 2998 2999 } 3000 3001 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) { 3002 assert(list != NULL, "list can't be null"); 3003 if (!list->is_empty()) { 3004 MutexLocker x(FreeList_lock, Mutex::_no_safepoint_check_flag); 3005 _hrm.insert_list_into_free_list(list); 3006 } 3007 } 3008 3009 void G1CollectedHeap::decrement_summary_bytes(size_t bytes) { 3010 decrease_used(bytes); 3011 } 3012 3013 void G1CollectedHeap::clear_eden() { 3014 _eden.clear(); 3015 } 3016 3017 void G1CollectedHeap::clear_collection_set() { 3018 collection_set()->clear(); 3019 } 3020 3021 void G1CollectedHeap::rebuild_free_region_list() { 3022 Ticks start = Ticks::now(); 3023 _hrm.rebuild_free_list(workers()); 3024 phase_times()->record_total_rebuild_freelist_time_ms((Ticks::now() - start).seconds() * 1000.0); 3025 } 3026 3027 class G1AbandonCollectionSetClosure : public HeapRegionClosure { 3028 public: 3029 virtual bool do_heap_region(HeapRegion* r) { 3030 assert(r->in_collection_set(), "Region %u must have been in collection set", r->hrm_index()); 3031 G1CollectedHeap::heap()->clear_region_attr(r); 3032 r->clear_young_index_in_cset(); 3033 return false; 3034 } 3035 }; 3036 3037 void G1CollectedHeap::abandon_collection_set(G1CollectionSet* collection_set) { 3038 G1AbandonCollectionSetClosure cl; 3039 collection_set_iterate_all(&cl); 3040 3041 collection_set->clear(); 3042 collection_set->stop_incremental_building(); 3043 } 3044 3045 bool G1CollectedHeap::is_old_gc_alloc_region(HeapRegion* hr) { 3046 return _allocator->is_retained_old_region(hr); 3047 } 3048 3049 void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { 3050 _eden.add(hr); 3051 _policy->set_region_eden(hr); 3052 } 3053 3054 #ifdef ASSERT 3055 3056 class NoYoungRegionsClosure: public HeapRegionClosure { 3057 private: 3058 bool _success; 3059 public: 3060 NoYoungRegionsClosure() : _success(true) { } 3061 bool do_heap_region(HeapRegion* r) { 3062 if (r->is_young()) { 3063 log_error(gc, verify)("Region [" PTR_FORMAT ", " PTR_FORMAT ") tagged as young", 3064 p2i(r->bottom()), p2i(r->end())); 3065 _success = false; 3066 } 3067 return false; 3068 } 3069 bool success() { return _success; } 3070 }; 3071 3072 bool G1CollectedHeap::check_young_list_empty() { 3073 bool ret = (young_regions_count() == 0); 3074 3075 NoYoungRegionsClosure closure; 3076 heap_region_iterate(&closure); 3077 ret = ret && closure.success(); 3078 3079 return ret; 3080 } 3081 3082 #endif // ASSERT 3083 3084 // Remove the given HeapRegion from the appropriate region set. 3085 void G1CollectedHeap::prepare_region_for_full_compaction(HeapRegion* hr) { 3086 if (hr->is_archive()) { 3087 _archive_set.remove(hr); 3088 } else if (hr->is_humongous()) { 3089 _humongous_set.remove(hr); 3090 } else if (hr->is_old()) { 3091 _old_set.remove(hr); 3092 } else if (hr->is_young()) { 3093 // Note that emptying the eden and survivor lists is postponed and instead 3094 // done as the first step when rebuilding the regions sets again. The reason 3095 // for this is that during a full GC string deduplication needs to know if 3096 // a collected region was young or old when the full GC was initiated. 3097 hr->uninstall_surv_rate_group(); 3098 } else { 3099 // We ignore free regions, we'll empty the free list afterwards. 3100 assert(hr->is_free(), "it cannot be another type"); 3101 } 3102 } 3103 3104 void G1CollectedHeap::increase_used(size_t bytes) { 3105 _summary_bytes_used += bytes; 3106 } 3107 3108 void G1CollectedHeap::decrease_used(size_t bytes) { 3109 assert(_summary_bytes_used >= bytes, 3110 "invariant: _summary_bytes_used: " SIZE_FORMAT " should be >= bytes: " SIZE_FORMAT, 3111 _summary_bytes_used, bytes); 3112 _summary_bytes_used -= bytes; 3113 } 3114 3115 void G1CollectedHeap::set_used(size_t bytes) { 3116 _summary_bytes_used = bytes; 3117 } 3118 3119 class RebuildRegionSetsClosure : public HeapRegionClosure { 3120 private: 3121 bool _free_list_only; 3122 3123 HeapRegionSet* _old_set; 3124 HeapRegionSet* _archive_set; 3125 HeapRegionSet* _humongous_set; 3126 3127 HeapRegionManager* _hrm; 3128 3129 size_t _total_used; 3130 3131 public: 3132 RebuildRegionSetsClosure(bool free_list_only, 3133 HeapRegionSet* old_set, 3134 HeapRegionSet* archive_set, 3135 HeapRegionSet* humongous_set, 3136 HeapRegionManager* hrm) : 3137 _free_list_only(free_list_only), _old_set(old_set), _archive_set(archive_set), 3138 _humongous_set(humongous_set), _hrm(hrm), _total_used(0) { 3139 assert(_hrm->num_free_regions() == 0, "pre-condition"); 3140 if (!free_list_only) { 3141 assert(_old_set->is_empty(), "pre-condition"); 3142 assert(_archive_set->is_empty(), "pre-condition"); 3143 assert(_humongous_set->is_empty(), "pre-condition"); 3144 } 3145 } 3146 3147 bool do_heap_region(HeapRegion* r) { 3148 if (r->is_empty()) { 3149 assert(r->rem_set()->is_empty(), "Empty regions should have empty remembered sets."); 3150 // Add free regions to the free list 3151 r->set_free(); 3152 _hrm->insert_into_free_list(r); 3153 } else if (!_free_list_only) { 3154 assert(r->rem_set()->is_empty(), "At this point remembered sets must have been cleared."); 3155 3156 if (r->is_humongous()) { 3157 _humongous_set->add(r); 3158 } else if (r->is_archive()) { 3159 _archive_set->add(r); 3160 } else { 3161 assert(r->is_young() || r->is_free() || r->is_old(), "invariant"); 3162 // We now move all (non-humongous, non-old, non-archive) regions to old gen, 3163 // and register them as such. 3164 r->move_to_old(); 3165 _old_set->add(r); 3166 } 3167 _total_used += r->used(); 3168 } 3169 3170 return false; 3171 } 3172 3173 size_t total_used() { 3174 return _total_used; 3175 } 3176 }; 3177 3178 void G1CollectedHeap::rebuild_region_sets(bool free_list_only) { 3179 assert_at_safepoint_on_vm_thread(); 3180 3181 if (!free_list_only) { 3182 _eden.clear(); 3183 _survivor.clear(); 3184 } 3185 3186 RebuildRegionSetsClosure cl(free_list_only, 3187 &_old_set, &_archive_set, &_humongous_set, 3188 &_hrm); 3189 heap_region_iterate(&cl); 3190 3191 if (!free_list_only) { 3192 set_used(cl.total_used()); 3193 assert(_archive_allocator == nullptr, "must be, should not contribute to used"); 3194 } 3195 assert_used_and_recalculate_used_equal(this); 3196 } 3197 3198 // Methods for the mutator alloc region 3199 3200 HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size, 3201 bool force, 3202 uint node_index) { 3203 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); 3204 bool should_allocate = policy()->should_allocate_mutator_region(); 3205 if (force || should_allocate) { 3206 HeapRegion* new_alloc_region = new_region(word_size, 3207 HeapRegionType::Eden, 3208 false /* do_expand */, 3209 node_index); 3210 if (new_alloc_region != NULL) { 3211 set_region_short_lived_locked(new_alloc_region); 3212 _hr_printer.alloc(new_alloc_region, !should_allocate); 3213 _verifier->check_bitmaps("Mutator Region Allocation", new_alloc_region); 3214 _policy->remset_tracker()->update_at_allocate(new_alloc_region); 3215 return new_alloc_region; 3216 } 3217 } 3218 return NULL; 3219 } 3220 3221 void G1CollectedHeap::retire_mutator_alloc_region(HeapRegion* alloc_region, 3222 size_t allocated_bytes) { 3223 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); 3224 assert(alloc_region->is_eden(), "all mutator alloc regions should be eden"); 3225 3226 collection_set()->add_eden_region(alloc_region); 3227 increase_used(allocated_bytes); 3228 _eden.add_used_bytes(allocated_bytes); 3229 _hr_printer.retire(alloc_region); 3230 3231 // We update the eden sizes here, when the region is retired, 3232 // instead of when it's allocated, since this is the point that its 3233 // used space has been recorded in _summary_bytes_used. 3234 monitoring_support()->update_eden_size(); 3235 } 3236 3237 // Methods for the GC alloc regions 3238 3239 bool G1CollectedHeap::has_more_regions(G1HeapRegionAttr dest) { 3240 if (dest.is_old()) { 3241 return true; 3242 } else { 3243 return survivor_regions_count() < policy()->max_survivor_regions(); 3244 } 3245 } 3246 3247 HeapRegion* G1CollectedHeap::new_gc_alloc_region(size_t word_size, G1HeapRegionAttr dest, uint node_index) { 3248 assert(FreeList_lock->owned_by_self(), "pre-condition"); 3249 3250 if (!has_more_regions(dest)) { 3251 return NULL; 3252 } 3253 3254 HeapRegionType type; 3255 if (dest.is_young()) { 3256 type = HeapRegionType::Survivor; 3257 } else { 3258 type = HeapRegionType::Old; 3259 } 3260 3261 HeapRegion* new_alloc_region = new_region(word_size, 3262 type, 3263 true /* do_expand */, 3264 node_index); 3265 3266 if (new_alloc_region != NULL) { 3267 if (type.is_survivor()) { 3268 new_alloc_region->set_survivor(); 3269 _survivor.add(new_alloc_region); 3270 _verifier->check_bitmaps("Survivor Region Allocation", new_alloc_region); 3271 register_new_survivor_region_with_region_attr(new_alloc_region); 3272 } else { 3273 new_alloc_region->set_old(); 3274 _verifier->check_bitmaps("Old Region Allocation", new_alloc_region); 3275 } 3276 _policy->remset_tracker()->update_at_allocate(new_alloc_region); 3277 register_region_with_region_attr(new_alloc_region); 3278 _hr_printer.alloc(new_alloc_region); 3279 return new_alloc_region; 3280 } 3281 return NULL; 3282 } 3283 3284 void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region, 3285 size_t allocated_bytes, 3286 G1HeapRegionAttr dest) { 3287 _bytes_used_during_gc += allocated_bytes; 3288 if (dest.is_old()) { 3289 old_set_add(alloc_region); 3290 } else { 3291 assert(dest.is_young(), "Retiring alloc region should be young (%d)", dest.type()); 3292 _survivor.add_used_bytes(allocated_bytes); 3293 } 3294 3295 bool const during_im = collector_state()->in_concurrent_start_gc(); 3296 if (during_im && allocated_bytes > 0) { 3297 _cm->root_regions()->add(alloc_region->next_top_at_mark_start(), alloc_region->top()); 3298 } 3299 _hr_printer.retire(alloc_region); 3300 } 3301 3302 HeapRegion* G1CollectedHeap::alloc_highest_free_region() { 3303 bool expanded = false; 3304 uint index = _hrm.find_highest_free(&expanded); 3305 3306 if (index != G1_NO_HRM_INDEX) { 3307 if (expanded) { 3308 log_debug(gc, ergo, heap)("Attempt heap expansion (requested address range outside heap bounds). region size: " SIZE_FORMAT "B", 3309 HeapRegion::GrainWords * HeapWordSize); 3310 } 3311 return _hrm.allocate_free_regions_starting_at(index, 1); 3312 } 3313 return NULL; 3314 } 3315 3316 void G1CollectedHeap::mark_evac_failure_object(const oop obj, uint worker_id) const { 3317 // All objects failing evacuation are live. What we'll do is 3318 // that we'll update the prev marking info so that they are 3319 // all under PTAMS and explicitly marked. 3320 _cm->par_mark_in_prev_bitmap(obj); 3321 } 3322 3323 // Optimized nmethod scanning 3324 3325 class RegisterNMethodOopClosure: public OopClosure { 3326 G1CollectedHeap* _g1h; 3327 nmethod* _nm; 3328 3329 public: 3330 RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : 3331 _g1h(g1h), _nm(nm) {} 3332 3333 void do_oop(oop* p) { 3334 oop heap_oop = RawAccess<>::oop_load(p); 3335 if (!CompressedOops::is_null(heap_oop)) { 3336 oop obj = CompressedOops::decode_not_null(heap_oop); 3337 HeapRegion* hr = _g1h->heap_region_containing(obj); 3338 assert(!hr->is_continues_humongous(), 3339 "trying to add code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT 3340 " starting at " HR_FORMAT, 3341 p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region())); 3342 3343 // HeapRegion::add_code_root_locked() avoids adding duplicate entries. 3344 hr->add_code_root_locked(_nm); 3345 } 3346 } 3347 3348 void do_oop(narrowOop* p) { ShouldNotReachHere(); } 3349 }; 3350 3351 class UnregisterNMethodOopClosure: public OopClosure { 3352 G1CollectedHeap* _g1h; 3353 nmethod* _nm; 3354 3355 public: 3356 UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : 3357 _g1h(g1h), _nm(nm) {} 3358 3359 void do_oop(oop* p) { 3360 oop heap_oop = RawAccess<>::oop_load(p); 3361 if (!CompressedOops::is_null(heap_oop)) { 3362 oop obj = CompressedOops::decode_not_null(heap_oop); 3363 HeapRegion* hr = _g1h->heap_region_containing(obj); 3364 assert(!hr->is_continues_humongous(), 3365 "trying to remove code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT 3366 " starting at " HR_FORMAT, 3367 p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region())); 3368 3369 hr->remove_code_root(_nm); 3370 } 3371 } 3372 3373 void do_oop(narrowOop* p) { ShouldNotReachHere(); } 3374 }; 3375 3376 void G1CollectedHeap::register_nmethod(nmethod* nm) { 3377 guarantee(nm != NULL, "sanity"); 3378 RegisterNMethodOopClosure reg_cl(this, nm); 3379 nm->oops_do(®_cl); 3380 } 3381 3382 void G1CollectedHeap::unregister_nmethod(nmethod* nm) { 3383 guarantee(nm != NULL, "sanity"); 3384 UnregisterNMethodOopClosure reg_cl(this, nm); 3385 nm->oops_do(®_cl, true); 3386 } 3387 3388 void G1CollectedHeap::update_used_after_gc(bool evacuation_failed) { 3389 if (evacuation_failed) { 3390 // Reset the G1EvacuationFailureALot counters and flags 3391 evac_failure_injector()->reset(); 3392 3393 set_used(recalculate_used()); 3394 3395 assert(_archive_allocator == nullptr, "must be, should not contribute to used"); 3396 } else { 3397 // The "used" of the the collection set have already been subtracted 3398 // when they were freed. Add in the bytes used. 3399 increase_used(_bytes_used_during_gc); 3400 } 3401 } 3402 3403 void G1CollectedHeap::reset_hot_card_cache() { 3404 _hot_card_cache->reset_hot_cache(); 3405 _hot_card_cache->set_use_cache(true); 3406 } 3407 3408 void G1CollectedHeap::purge_code_root_memory() { 3409 G1CodeRootSet::purge(); 3410 } 3411 3412 class RebuildCodeRootClosure: public CodeBlobClosure { 3413 G1CollectedHeap* _g1h; 3414 3415 public: 3416 RebuildCodeRootClosure(G1CollectedHeap* g1h) : 3417 _g1h(g1h) {} 3418 3419 void do_code_blob(CodeBlob* cb) { 3420 nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL; 3421 if (nm == NULL) { 3422 return; 3423 } 3424 3425 _g1h->register_nmethod(nm); 3426 } 3427 }; 3428 3429 void G1CollectedHeap::rebuild_code_roots() { 3430 RebuildCodeRootClosure blob_cl(this); 3431 CodeCache::blobs_do(&blob_cl); 3432 } 3433 3434 void G1CollectedHeap::initialize_serviceability() { 3435 _monitoring_support->initialize_serviceability(); 3436 } 3437 3438 MemoryUsage G1CollectedHeap::memory_usage() { 3439 return _monitoring_support->memory_usage(); 3440 } 3441 3442 GrowableArray<GCMemoryManager*> G1CollectedHeap::memory_managers() { 3443 return _monitoring_support->memory_managers(); 3444 } 3445 3446 GrowableArray<MemoryPool*> G1CollectedHeap::memory_pools() { 3447 return _monitoring_support->memory_pools(); 3448 } 3449 3450 void G1CollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) { 3451 HeapRegion* region = heap_region_containing(start); 3452 region->fill_with_dummy_object(start, pointer_delta(end, start), zap); 3453 }