1 /*
2 * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "gc/parallel/objectStartArray.inline.hpp"
26 #include "gc/parallel/parallelArguments.hpp"
27 #include "gc/parallel/parallelInitLogger.hpp"
28 #include "gc/parallel/parallelScavengeHeap.inline.hpp"
29 #include "gc/parallel/psAdaptiveSizePolicy.hpp"
30 #include "gc/parallel/psMemoryPool.hpp"
31 #include "gc/parallel/psParallelCompact.inline.hpp"
32 #include "gc/parallel/psPromotionManager.hpp"
33 #include "gc/parallel/psScavenge.hpp"
34 #include "gc/parallel/psVMOperations.hpp"
35 #include "gc/shared/barrierSetNMethod.hpp"
36 #include "gc/shared/fullGCForwarding.inline.hpp"
37 #include "gc/shared/gcHeapSummary.hpp"
38 #include "gc/shared/gcLocker.inline.hpp"
39 #include "gc/shared/gcWhen.hpp"
40 #include "gc/shared/genArguments.hpp"
41 #include "gc/shared/locationPrinter.inline.hpp"
42 #include "gc/shared/scavengableNMethods.hpp"
43 #include "gc/shared/suspendibleThreadSet.hpp"
44 #include "logging/log.hpp"
45 #include "memory/iterator.hpp"
46 #include "memory/metaspaceCounters.hpp"
47 #include "memory/metaspaceUtils.hpp"
48 #include "memory/reservedSpace.hpp"
49 #include "memory/universe.hpp"
50 #include "oops/oop.inline.hpp"
51 #include "runtime/atomic.hpp"
52 #include "runtime/cpuTimeCounters.hpp"
53 #include "runtime/globals_extension.hpp"
54 #include "runtime/handles.inline.hpp"
55 #include "runtime/init.hpp"
56 #include "runtime/java.hpp"
57 #include "runtime/vmThread.hpp"
58 #include "services/memoryManager.hpp"
59 #include "utilities/macros.hpp"
60 #include "utilities/vmError.hpp"
61
62 PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = nullptr;
63 GCPolicyCounters* ParallelScavengeHeap::_gc_policy_counters = nullptr;
64 size_t ParallelScavengeHeap::_desired_page_size = 0;
65
66 jint ParallelScavengeHeap::initialize() {
67 const size_t reserved_heap_size = ParallelArguments::heap_reserved_size_bytes();
68
69 assert(_desired_page_size != 0, "Should be initialized");
70 ReservedHeapSpace heap_rs = Universe::reserve_heap(reserved_heap_size, HeapAlignment, _desired_page_size);
71 // Adjust SpaceAlignment based on actually used large page size.
72 if (UseLargePages) {
73 SpaceAlignment = MAX2(heap_rs.page_size(), default_space_alignment());
74 }
75 assert(is_aligned(SpaceAlignment, heap_rs.page_size()), "inv");
76
77 trace_actual_reserved_page_size(reserved_heap_size, heap_rs);
78
79 initialize_reserved_region(heap_rs);
80 // Layout the reserved space for the generations.
81 ReservedSpace old_rs = heap_rs.first_part(MaxOldSize, SpaceAlignment);
82 ReservedSpace young_rs = heap_rs.last_part(MaxOldSize, SpaceAlignment);
83 assert(young_rs.size() == MaxNewSize, "Didn't reserve all of the heap");
84
85 PSCardTable* card_table = new PSCardTable(_reserved);
86 card_table->initialize(old_rs.base(), young_rs.base());
87
88 CardTableBarrierSet* const barrier_set = new CardTableBarrierSet(card_table);
89 BarrierSet::set_barrier_set(barrier_set);
90
91 // Set up WorkerThreads
92 _workers.initialize_workers();
93
94 // Create and initialize the generations.
95 _young_gen = new PSYoungGen(
96 young_rs,
97 NewSize,
98 MinNewSize,
99 MaxNewSize);
100 _old_gen = new PSOldGen(
101 old_rs,
102 OldSize,
103 MinOldSize,
104 MaxOldSize);
105
106 assert(young_gen()->max_gen_size() == young_rs.size(),"Consistency check");
107 assert(old_gen()->max_gen_size() == old_rs.size(), "Consistency check");
108
109 double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0;
110
111 _size_policy = new PSAdaptiveSizePolicy(SpaceAlignment,
112 max_gc_pause_sec);
113
114 assert((old_gen()->virtual_space()->high_boundary() ==
115 young_gen()->virtual_space()->low_boundary()),
116 "Boundaries must meet");
117 // initialize the policy counters - 2 collectors, 2 generations
118 _gc_policy_counters = new GCPolicyCounters("ParScav:MSC", 2, 2);
119
120 if (!PSParallelCompact::initialize_aux_data()) {
121 return JNI_ENOMEM;
122 }
123
124 // Create CPU time counter
125 CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
126
127 ParallelInitLogger::print();
128
129 FullGCForwarding::initialize(_reserved);
130
131 return JNI_OK;
132 }
133
134 void ParallelScavengeHeap::initialize_serviceability() {
135
136 _eden_pool = new PSEdenSpacePool(_young_gen,
137 _young_gen->eden_space(),
138 "PS Eden Space",
139 false /* support_usage_threshold */);
140
141 _survivor_pool = new PSSurvivorSpacePool(_young_gen,
142 "PS Survivor Space",
143 false /* support_usage_threshold */);
144
145 _old_pool = new PSOldGenerationPool(_old_gen,
146 "PS Old Gen",
147 true /* support_usage_threshold */);
148
149 _young_manager = new GCMemoryManager("PS Scavenge");
150 _old_manager = new GCMemoryManager("PS MarkSweep");
151
152 _old_manager->add_pool(_eden_pool);
153 _old_manager->add_pool(_survivor_pool);
154 _old_manager->add_pool(_old_pool);
155
156 _young_manager->add_pool(_eden_pool);
157 _young_manager->add_pool(_survivor_pool);
158
159 }
160
161 class PSIsScavengable : public BoolObjectClosure {
162 bool do_object_b(oop obj) {
163 return ParallelScavengeHeap::heap()->is_in_young(obj);
164 }
165 };
166
167 static PSIsScavengable _is_scavengable;
168
169 void ParallelScavengeHeap::post_initialize() {
170 CollectedHeap::post_initialize();
171 // Need to init the tenuring threshold
172 PSScavenge::initialize();
173 PSParallelCompact::post_initialize();
174 PSPromotionManager::initialize();
175
176 ScavengableNMethods::initialize(&_is_scavengable);
177 GCLocker::initialize();
178 }
179
180 void ParallelScavengeHeap::gc_epilogue(bool full) {
181 if (_is_heap_almost_full) {
182 // Reset emergency state if eden is empty after a young/full gc
183 if (_young_gen->eden_space()->is_empty()) {
184 log_debug(gc)("Leaving memory constrained state; back to normal");
185 _is_heap_almost_full = false;
186 }
187 } else {
188 if (full && !_young_gen->eden_space()->is_empty()) {
189 log_debug(gc)("Non-empty young-gen after full-gc; in memory constrained state");
190 _is_heap_almost_full = true;
191 }
192 }
193 }
194
195 void ParallelScavengeHeap::update_counters() {
196 young_gen()->update_counters();
197 old_gen()->update_counters();
198 MetaspaceCounters::update_performance_counters();
199 update_parallel_worker_threads_cpu_time();
200 }
201
202 size_t ParallelScavengeHeap::capacity() const {
203 size_t value = young_gen()->capacity_in_bytes() + old_gen()->capacity_in_bytes();
204 return value;
205 }
206
207 size_t ParallelScavengeHeap::used() const {
208 size_t value = young_gen()->used_in_bytes() + old_gen()->used_in_bytes();
209 return value;
210 }
211
212 size_t ParallelScavengeHeap::max_capacity() const {
213 size_t estimated = reserved_region().byte_size();
214 if (UseAdaptiveSizePolicy) {
215 estimated -= _size_policy->max_survivor_size(young_gen()->max_gen_size());
216 } else {
217 estimated -= young_gen()->to_space()->capacity_in_bytes();
218 }
219 return MAX2(estimated, capacity());
220 }
221
222 bool ParallelScavengeHeap::is_in(const void* p) const {
223 return young_gen()->is_in(p) || old_gen()->is_in(p);
224 }
225
226 bool ParallelScavengeHeap::is_in_reserved(const void* p) const {
227 return young_gen()->is_in_reserved(p) || old_gen()->is_in_reserved(p);
228 }
229
230 bool ParallelScavengeHeap::requires_barriers(stackChunkOop p) const {
231 return !is_in_young(p);
232 }
233
234 // There are two levels of allocation policy here.
235 //
236 // When an allocation request fails, the requesting thread must invoke a VM
237 // operation, transfer control to the VM thread, and await the results of a
238 // garbage collection. That is quite expensive, and we should avoid doing it
239 // multiple times if possible.
240 //
241 // To accomplish this, we have a basic allocation policy, and also a
242 // failed allocation policy.
243 //
244 // The basic allocation policy controls how you allocate memory without
245 // attempting garbage collection. It is okay to grab locks and
246 // expand the heap, if that can be done without coming to a safepoint.
247 // It is likely that the basic allocation policy will not be very
248 // aggressive.
249 //
250 // The failed allocation policy is invoked from the VM thread after
251 // the basic allocation policy is unable to satisfy a mem_allocate
252 // request. This policy needs to cover the entire range of collection,
253 // heap expansion, and out-of-memory conditions. It should make every
254 // attempt to allocate the requested memory.
255
256 // Basic allocation policy. Should never be called at a safepoint, or
257 // from the VM thread.
258 //
259 // This method must handle cases where many mem_allocate requests fail
260 // simultaneously. When that happens, only one VM operation will succeed,
261 // and the rest will not be executed. For that reason, this method loops
262 // during failed allocation attempts. If the java heap becomes exhausted,
263 // we rely on the size_policy object to force a bail out.
264 HeapWord* ParallelScavengeHeap::mem_allocate(size_t size) {
265 assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint");
266 assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread");
267 assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");
268
269 bool is_tlab = false;
270 return mem_allocate_work(size, is_tlab);
271 }
272
273 HeapWord* ParallelScavengeHeap::mem_allocate_cas_noexpand(size_t size, bool is_tlab) {
274 // Try young-gen first.
275 HeapWord* result = young_gen()->allocate(size);
276 if (result != nullptr) {
277 return result;
278 }
279
280 // Try allocating from the old gen for non-TLAB and large allocations.
281 if (!is_tlab) {
282 if (!should_alloc_in_eden(size)) {
283 result = old_gen()->cas_allocate_noexpand(size);
284 if (result != nullptr) {
285 return result;
286 }
287 }
288 }
289
290 // In extreme cases, try allocating in from space also.
291 if (_is_heap_almost_full) {
292 result = young_gen()->from_space()->cas_allocate(size);
293 if (result != nullptr) {
294 return result;
295 }
296 if (!is_tlab) {
297 result = old_gen()->cas_allocate_noexpand(size);
298 if (result != nullptr) {
299 return result;
300 }
301 }
302 }
303
304 return nullptr;
305 }
306
307 HeapWord* ParallelScavengeHeap::mem_allocate_work(size_t size, bool is_tlab) {
308 for (uint loop_count = 0; /* empty */; ++loop_count) {
309 HeapWord* result;
310 {
311 ConditionalMutexLocker locker(Heap_lock, !is_init_completed());
312 result = mem_allocate_cas_noexpand(size, is_tlab);
313 if (result != nullptr) {
314 return result;
315 }
316 }
317
318 // Read total_collections() under the lock so that multiple
319 // allocation-failures result in one GC.
320 uint gc_count;
321 {
322 MutexLocker ml(Heap_lock);
323
324 // Re-try after acquiring the lock, because a GC might have occurred
325 // while waiting for this lock.
326 result = mem_allocate_cas_noexpand(size, is_tlab);
327 if (result != nullptr) {
328 return result;
329 }
330
331 if (!is_init_completed()) {
332 // Double checked locking, this ensure that is_init_completed() does not
333 // transition while expanding the heap.
334 MonitorLocker ml(InitCompleted_lock, Monitor::_no_safepoint_check_flag);
335 if (!is_init_completed()) {
336 // Can't do GC; try heap expansion to satisfy the request.
337 result = expand_heap_and_allocate(size, is_tlab);
338 if (result != nullptr) {
339 return result;
340 }
341 }
342 }
343
344 gc_count = total_collections();
345 }
346
347 {
348 VM_ParallelCollectForAllocation op(size, is_tlab, gc_count);
349 VMThread::execute(&op);
350
351 if (op.gc_succeeded()) {
352 assert(is_in_or_null(op.result()), "result not in heap");
353 return op.result();
354 }
355 }
356
357 // Was the gc-overhead reached inside the safepoint? If so, this mutator
358 // should return null as well for global consistency.
359 if (_gc_overhead_counter >= GCOverheadLimitThreshold) {
360 return nullptr;
361 }
362
363 if ((QueuedAllocationWarningCount > 0) &&
364 (loop_count % QueuedAllocationWarningCount == 0)) {
365 log_warning(gc)("ParallelScavengeHeap::mem_allocate retries %d times, size=%zu", loop_count, size);
366 }
367 }
368 }
369
370 void ParallelScavengeHeap::do_full_collection(bool clear_all_soft_refs) {
371 // No need for max-compaction in this context.
372 const bool should_do_max_compaction = false;
373 PSParallelCompact::invoke(clear_all_soft_refs, should_do_max_compaction);
374 }
375
376 bool ParallelScavengeHeap::should_attempt_young_gc() const {
377 const bool ShouldRunYoungGC = true;
378 const bool ShouldRunFullGC = false;
379
380 if (!_young_gen->to_space()->is_empty()) {
381 log_debug(gc, ergo)("To-space is not empty; run full-gc instead.");
382 return ShouldRunFullGC;
383 }
384
385 // Check if the predicted promoted bytes will overflow free space in old-gen.
386 PSAdaptiveSizePolicy* policy = _size_policy;
387
388 size_t avg_promoted = (size_t) policy->padded_average_promoted_in_bytes();
389 size_t promotion_estimate = MIN2(avg_promoted, _young_gen->used_in_bytes());
390 // Total free size after possible old gen expansion
391 size_t free_in_old_gen_with_expansion = _old_gen->max_gen_size() - _old_gen->used_in_bytes();
392
393 log_trace(gc, ergo)("average_promoted %zu; padded_average_promoted %zu",
394 (size_t) policy->average_promoted_in_bytes(),
395 (size_t) policy->padded_average_promoted_in_bytes());
396
397 if (promotion_estimate >= free_in_old_gen_with_expansion) {
398 log_debug(gc, ergo)("Run full-gc; predicted promotion size >= max free space in old-gen: %zu >= %zu",
399 promotion_estimate, free_in_old_gen_with_expansion);
400 return ShouldRunFullGC;
401 }
402
403 if (UseAdaptiveSizePolicy) {
404 // Also checking OS has enough free memory to commit and expand old-gen.
405 // Otherwise, the recorded gc-pause-time might be inflated to include time
406 // of OS preparing free memory, resulting in inaccurate young-gen resizing.
407 assert(_old_gen->committed().byte_size() >= _old_gen->used_in_bytes(), "inv");
408 // Use uint64_t instead of size_t for 32bit compatibility.
409 uint64_t free_mem_in_os;
410 if (os::free_memory(free_mem_in_os)) {
411 size_t actual_free = (size_t)MIN2(_old_gen->committed().byte_size() - _old_gen->used_in_bytes() + free_mem_in_os,
412 (uint64_t)SIZE_MAX);
413 if (promotion_estimate > actual_free) {
414 log_debug(gc, ergo)("Run full-gc; predicted promotion size > free space in old-gen and OS: %zu > %zu",
415 promotion_estimate, actual_free);
416 return ShouldRunFullGC;
417 }
418 }
419 }
420
421 // No particular reasons to run full-gc, so young-gc.
422 return ShouldRunYoungGC;
423 }
424
425 static bool check_gc_heap_free_limit(size_t free_bytes, size_t capacity_bytes) {
426 return (free_bytes * 100 / capacity_bytes) < GCHeapFreeLimit;
427 }
428
429 bool ParallelScavengeHeap::check_gc_overhead_limit() {
430 assert(SafepointSynchronize::is_at_safepoint(), "precondition");
431
432 if (UseGCOverheadLimit) {
433 // The goal here is to return null prematurely so that apps can exit
434 // gracefully when GC takes the most time.
435 bool little_mutator_time = _size_policy->mutator_time_percent() * 100 < (100 - GCTimeLimit);
436 bool little_free_space = check_gc_heap_free_limit(_young_gen->free_in_bytes(), _young_gen->capacity_in_bytes())
437 && check_gc_heap_free_limit( _old_gen->free_in_bytes(), _old_gen->capacity_in_bytes());
438
439 log_debug(gc)("GC Overhead Limit: GC Time %f Free Space Young %f Old %f Counter %zu",
440 (100 - _size_policy->mutator_time_percent()),
441 percent_of(_young_gen->free_in_bytes(), _young_gen->capacity_in_bytes()),
442 percent_of(_old_gen->free_in_bytes(), _old_gen->capacity_in_bytes()),
443 _gc_overhead_counter);
444
445 if (little_mutator_time && little_free_space) {
446 _gc_overhead_counter++;
447 if (_gc_overhead_counter >= GCOverheadLimitThreshold) {
448 return true;
449 }
450 } else {
451 _gc_overhead_counter = 0;
452 }
453 }
454 return false;
455 }
456
457 HeapWord* ParallelScavengeHeap::expand_heap_and_allocate(size_t size, bool is_tlab) {
458 #ifdef ASSERT
459 assert(Heap_lock->is_locked(), "precondition");
460 if (is_init_completed()) {
461 assert(SafepointSynchronize::is_at_safepoint(), "precondition");
462 assert(Thread::current()->is_VM_thread(), "precondition");
463 } else {
464 assert(Thread::current()->is_Java_thread(), "precondition");
465 assert(Heap_lock->owned_by_self(), "precondition");
466 }
467 #endif
468
469 HeapWord* result = young_gen()->expand_and_allocate(size);
470
471 if (result == nullptr && !is_tlab) {
472 result = old_gen()->expand_and_allocate(size);
473 }
474
475 return result; // Could be null if we are out of space.
476 }
477
478 HeapWord* ParallelScavengeHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {
479 assert(size != 0, "precondition");
480
481 HeapWord* result = nullptr;
482
483 if (!_is_heap_almost_full) {
484 // If young-gen can handle this allocation, attempt young-gc firstly, as young-gc is usually cheaper.
485 bool should_run_young_gc = is_tlab || should_alloc_in_eden(size);
486
487 collect_at_safepoint(!should_run_young_gc);
488
489 // If gc-overhead is reached, we will skip allocation.
490 if (!check_gc_overhead_limit()) {
491 result = expand_heap_and_allocate(size, is_tlab);
492 if (result != nullptr) {
493 return result;
494 }
495 }
496 }
497
498 // Last resort GC; clear soft refs and do max-compaction before throwing OOM.
499 {
500 const bool clear_all_soft_refs = true;
501 const bool should_do_max_compaction = true;
502 PSParallelCompact::invoke(clear_all_soft_refs, should_do_max_compaction);
503 }
504
505 if (check_gc_overhead_limit()) {
506 log_info(gc)("GC Overhead Limit exceeded too often (%zu).", GCOverheadLimitThreshold);
507 return nullptr;
508 }
509
510 result = expand_heap_and_allocate(size, is_tlab);
511
512 return result;
513 }
514
515 void ParallelScavengeHeap::ensure_parsability(bool retire_tlabs) {
516 CollectedHeap::ensure_parsability(retire_tlabs);
517 young_gen()->eden_space()->ensure_parsability();
518 }
519
520 size_t ParallelScavengeHeap::tlab_capacity() const {
521 return young_gen()->eden_space()->tlab_capacity();
522 }
523
524 size_t ParallelScavengeHeap::tlab_used() const {
525 return young_gen()->eden_space()->tlab_used();
526 }
527
528 size_t ParallelScavengeHeap::unsafe_max_tlab_alloc() const {
529 return young_gen()->eden_space()->unsafe_max_tlab_alloc();
530 }
531
532 HeapWord* ParallelScavengeHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) {
533 HeapWord* result = mem_allocate_work(requested_size /* size */,
534 true /* is_tlab */);
535 if (result != nullptr) {
536 *actual_size = requested_size;
537 }
538
539 return result;
540 }
541
542 void ParallelScavengeHeap::resize_all_tlabs() {
543 CollectedHeap::resize_all_tlabs();
544 }
545
546 void ParallelScavengeHeap::prune_scavengable_nmethods() {
547 ScavengableNMethods::prune_nmethods_not_into_young();
548 }
549
550 void ParallelScavengeHeap::prune_unlinked_nmethods() {
551 ScavengableNMethods::prune_unlinked_nmethods();
552 }
553
554 void ParallelScavengeHeap::collect(GCCause::Cause cause) {
555 assert(!Heap_lock->owned_by_self(),
556 "this thread should not own the Heap_lock");
557
558 uint gc_count = 0;
559 uint full_gc_count = 0;
560 {
561 MutexLocker ml(Heap_lock);
562 // This value is guarded by the Heap_lock
563 gc_count = total_collections();
564 full_gc_count = total_full_collections();
565 }
566
567 VM_ParallelGCCollect op(gc_count, full_gc_count, cause);
568 VMThread::execute(&op);
569 }
570
571 void ParallelScavengeHeap::collect_at_safepoint(bool is_full) {
572 assert(!GCLocker::is_active(), "precondition");
573 bool clear_soft_refs = GCCause::should_clear_all_soft_refs(_gc_cause);
574
575 if (!is_full && should_attempt_young_gc()) {
576 bool young_gc_success = PSScavenge::invoke(clear_soft_refs);
577 if (young_gc_success) {
578 return;
579 }
580 log_debug(gc, heap)("Upgrade to Full-GC since Young-gc failed.");
581 }
582
583 const bool should_do_max_compaction = false;
584 PSParallelCompact::invoke(clear_soft_refs, should_do_max_compaction);
585 }
586
587 void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) {
588 young_gen()->object_iterate(cl);
589 old_gen()->object_iterate(cl);
590 }
591
592 // The HeapBlockClaimer is used during parallel iteration over the heap,
593 // allowing workers to claim heap areas ("blocks"), gaining exclusive rights to these.
594 // The eden and survivor spaces are treated as single blocks as it is hard to divide
595 // these spaces.
596 // The old space is divided into fixed-size blocks.
597 class HeapBlockClaimer : public StackObj {
598 Atomic<size_t> _claimed_index;
599
600 public:
601 static const size_t InvalidIndex = SIZE_MAX;
602 static const size_t EdenIndex = 0;
603 static const size_t SurvivorIndex = 1;
604 static const size_t NumNonOldGenClaims = 2;
605
606 HeapBlockClaimer() : _claimed_index(EdenIndex) { }
607 // Claim the block and get the block index.
608 size_t claim_and_get_block() {
609 size_t block_index;
610 block_index = _claimed_index.fetch_then_add(1u);
611
612 PSOldGen* old_gen = ParallelScavengeHeap::heap()->old_gen();
613 size_t num_claims = old_gen->num_iterable_blocks() + NumNonOldGenClaims;
614
615 return block_index < num_claims ? block_index : InvalidIndex;
616 }
617 };
618
619 void ParallelScavengeHeap::object_iterate_parallel(ObjectClosure* cl,
620 HeapBlockClaimer* claimer) {
621 size_t block_index = claimer->claim_and_get_block();
622 // Iterate until all blocks are claimed
623 if (block_index == HeapBlockClaimer::EdenIndex) {
624 young_gen()->eden_space()->object_iterate(cl);
625 block_index = claimer->claim_and_get_block();
626 }
627 if (block_index == HeapBlockClaimer::SurvivorIndex) {
628 young_gen()->from_space()->object_iterate(cl);
629 young_gen()->to_space()->object_iterate(cl);
630 block_index = claimer->claim_and_get_block();
631 }
632 while (block_index != HeapBlockClaimer::InvalidIndex) {
633 old_gen()->object_iterate_block(cl, block_index - HeapBlockClaimer::NumNonOldGenClaims);
634 block_index = claimer->claim_and_get_block();
635 }
636 }
637
638 class PSScavengeParallelObjectIterator : public ParallelObjectIteratorImpl {
639 private:
640 ParallelScavengeHeap* _heap;
641 HeapBlockClaimer _claimer;
642
643 public:
644 PSScavengeParallelObjectIterator() :
645 _heap(ParallelScavengeHeap::heap()),
646 _claimer() {}
647
648 virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
649 _heap->object_iterate_parallel(cl, &_claimer);
650 }
651 };
652
653 ParallelObjectIteratorImpl* ParallelScavengeHeap::parallel_object_iterator(uint thread_num) {
654 return new PSScavengeParallelObjectIterator();
655 }
656
657 HeapWord* ParallelScavengeHeap::block_start(const void* addr) const {
658 if (young_gen()->is_in_reserved(addr)) {
659 assert(young_gen()->is_in(addr),
660 "addr should be in allocated part of young gen");
661 // called from os::print_location by find or VMError
662 if (DebuggingContext::is_enabled() || VMError::is_error_reported()) {
663 return nullptr;
664 }
665 Unimplemented();
666 } else if (old_gen()->is_in_reserved(addr)) {
667 assert(old_gen()->is_in(addr),
668 "addr should be in allocated part of old gen");
669 return old_gen()->start_array()->object_start((HeapWord*)addr);
670 }
671 return nullptr;
672 }
673
674 bool ParallelScavengeHeap::block_is_obj(const HeapWord* addr) const {
675 return block_start(addr) == addr;
676 }
677
678 void ParallelScavengeHeap::prepare_for_verify() {
679 ensure_parsability(false); // no need to retire TLABs for verification
680 }
681
682 PSHeapSummary ParallelScavengeHeap::create_ps_heap_summary() {
683 PSOldGen* old = old_gen();
684 HeapWord* old_committed_end = (HeapWord*)old->virtual_space()->committed_high_addr();
685 HeapWord* old_reserved_start = old->reserved().start();
686 HeapWord* old_reserved_end = old->reserved().end();
687 VirtualSpaceSummary old_summary(old_reserved_start, old_committed_end, old_reserved_end);
688 SpaceSummary old_space(old_reserved_start, old_committed_end, old->used_in_bytes());
689
690 PSYoungGen* young = young_gen();
691 VirtualSpaceSummary young_summary(young->reserved().start(),
692 (HeapWord*)young->virtual_space()->committed_high_addr(), young->reserved().end());
693
694 MutableSpace* eden = young_gen()->eden_space();
695 SpaceSummary eden_space(eden->bottom(), eden->end(), eden->used_in_bytes());
696
697 MutableSpace* from = young_gen()->from_space();
698 SpaceSummary from_space(from->bottom(), from->end(), from->used_in_bytes());
699
700 MutableSpace* to = young_gen()->to_space();
701 SpaceSummary to_space(to->bottom(), to->end(), to->used_in_bytes());
702
703 VirtualSpaceSummary heap_summary = create_heap_space_summary();
704 return PSHeapSummary(heap_summary, used(), old_summary, old_space, young_summary, eden_space, from_space, to_space);
705 }
706
707 bool ParallelScavengeHeap::print_location(outputStream* st, void* addr) const {
708 return BlockLocationPrinter<ParallelScavengeHeap>::print_location(st, addr);
709 }
710
711 void ParallelScavengeHeap::print_heap_on(outputStream* st) const {
712 if (young_gen() != nullptr) {
713 young_gen()->print_on(st);
714 }
715 if (old_gen() != nullptr) {
716 old_gen()->print_on(st);
717 }
718 }
719
720 void ParallelScavengeHeap::print_gc_on(outputStream* st) const {
721 BarrierSet* bs = BarrierSet::barrier_set();
722 if (bs != nullptr) {
723 bs->print_on(st);
724 }
725 st->cr();
726
727 PSParallelCompact::print_on(st);
728 }
729
730 void ParallelScavengeHeap::gc_threads_do(ThreadClosure* tc) const {
731 ParallelScavengeHeap::heap()->workers().threads_do(tc);
732 }
733
734 void ParallelScavengeHeap::print_tracing_info() const {
735 log_debug(gc, heap, exit)("Accumulated young generation GC time %3.7f secs", PSScavenge::accumulated_time()->seconds());
736 log_debug(gc, heap, exit)("Accumulated old generation GC time %3.7f secs", PSParallelCompact::accumulated_time()->seconds());
737 }
738
739 PreGenGCValues ParallelScavengeHeap::get_pre_gc_values() const {
740 const PSYoungGen* const young = young_gen();
741 const MutableSpace* const eden = young->eden_space();
742 const MutableSpace* const from = young->from_space();
743 const PSOldGen* const old = old_gen();
744
745 return PreGenGCValues(young->used_in_bytes(),
746 young->capacity_in_bytes(),
747 eden->used_in_bytes(),
748 eden->capacity_in_bytes(),
749 from->used_in_bytes(),
750 from->capacity_in_bytes(),
751 old->used_in_bytes(),
752 old->capacity_in_bytes());
753 }
754
755 void ParallelScavengeHeap::print_heap_change(const PreGenGCValues& pre_gc_values) const {
756 const PSYoungGen* const young = young_gen();
757 const MutableSpace* const eden = young->eden_space();
758 const MutableSpace* const from = young->from_space();
759 const PSOldGen* const old = old_gen();
760
761 log_info(gc, heap)(HEAP_CHANGE_FORMAT" "
762 HEAP_CHANGE_FORMAT" "
763 HEAP_CHANGE_FORMAT,
764 HEAP_CHANGE_FORMAT_ARGS(young->name(),
765 pre_gc_values.young_gen_used(),
766 pre_gc_values.young_gen_capacity(),
767 young->used_in_bytes(),
768 young->capacity_in_bytes()),
769 HEAP_CHANGE_FORMAT_ARGS("Eden",
770 pre_gc_values.eden_used(),
771 pre_gc_values.eden_capacity(),
772 eden->used_in_bytes(),
773 eden->capacity_in_bytes()),
774 HEAP_CHANGE_FORMAT_ARGS("From",
775 pre_gc_values.from_used(),
776 pre_gc_values.from_capacity(),
777 from->used_in_bytes(),
778 from->capacity_in_bytes()));
779 log_info(gc, heap)(HEAP_CHANGE_FORMAT,
780 HEAP_CHANGE_FORMAT_ARGS(old->name(),
781 pre_gc_values.old_gen_used(),
782 pre_gc_values.old_gen_capacity(),
783 old->used_in_bytes(),
784 old->capacity_in_bytes()));
785 MetaspaceUtils::print_metaspace_change(pre_gc_values.metaspace_sizes());
786 }
787
788 void ParallelScavengeHeap::verify(VerifyOption option /* ignored */) {
789 log_debug(gc, verify)("Tenured");
790 old_gen()->verify();
791
792 log_debug(gc, verify)("Eden");
793 young_gen()->verify();
794
795 log_debug(gc, verify)("CardTable");
796 card_table()->verify_all_young_refs_imprecise();
797 }
798
799 void ParallelScavengeHeap::trace_actual_reserved_page_size(const size_t reserved_heap_size, const ReservedSpace rs) {
800 // Check if Info level is enabled, since os::trace_page_sizes() logs on Info level.
801 if(log_is_enabled(Info, pagesize)) {
802 const size_t page_size = rs.page_size();
803 os::trace_page_sizes("Heap",
804 MinHeapSize,
805 reserved_heap_size,
806 rs.base(),
807 rs.size(),
808 page_size);
809 }
810 }
811
812 void ParallelScavengeHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
813 const PSHeapSummary& heap_summary = create_ps_heap_summary();
814 gc_tracer->report_gc_heap_summary(when, heap_summary);
815
816 const MetaspaceSummary& metaspace_summary = create_metaspace_summary();
817 gc_tracer->report_metaspace_summary(when, metaspace_summary);
818 }
819
820 CardTableBarrierSet* ParallelScavengeHeap::barrier_set() {
821 return barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
822 }
823
824 PSCardTable* ParallelScavengeHeap::card_table() {
825 return static_cast<PSCardTable*>(barrier_set()->card_table());
826 }
827
828 static size_t calculate_free_from_free_ratio_flag(size_t live, uintx free_percent) {
829 assert(free_percent != 100, "precondition");
830 // We want to calculate how much free memory there can be based on the
831 // live size.
832 // percent * (free + live) = free
833 // =>
834 // free = (live * percent) / (1 - percent)
835
836 const double percent = free_percent / 100.0;
837 return live * percent / (1.0 - percent);
838 }
839
840 size_t ParallelScavengeHeap::calculate_desired_old_gen_capacity(size_t old_gen_live_size) {
841 // If min free percent is 100%, the old-gen should always be in its max capacity
842 if (MinHeapFreeRatio == 100) {
843 return _old_gen->max_gen_size();
844 }
845
846 // Using recorded data to calculate the new capacity of old-gen to avoid
847 // excessive expansion but also keep footprint low
848
849 size_t promoted_estimate = _size_policy->padded_average_promoted_in_bytes();
850 // Should have at least this free room for the next young-gc promotion.
851 size_t free_size = promoted_estimate;
852
853 size_t largest_live_size = MAX2((size_t)_size_policy->peak_old_gen_used_estimate(), old_gen_live_size);
854 free_size += largest_live_size - old_gen_live_size;
855
856 // Respect free percent
857 if (MinHeapFreeRatio != 0) {
858 size_t min_free = calculate_free_from_free_ratio_flag(old_gen_live_size, MinHeapFreeRatio);
859 free_size = MAX2(free_size, min_free);
860 }
861
862 if (MaxHeapFreeRatio != 100) {
863 size_t max_free = calculate_free_from_free_ratio_flag(old_gen_live_size, MaxHeapFreeRatio);
864 free_size = MIN2(max_free, free_size);
865 }
866
867 return old_gen_live_size + free_size;
868 }
869
870 void ParallelScavengeHeap::resize_old_gen_after_full_gc() {
871 size_t current_capacity = _old_gen->capacity_in_bytes();
872 size_t desired_capacity = calculate_desired_old_gen_capacity(old_gen()->used_in_bytes());
873
874 // If MinHeapFreeRatio is at its default value; shrink cautiously. Otherwise, users expect prompt shrinking.
875 if (FLAG_IS_DEFAULT(MinHeapFreeRatio)) {
876 if (desired_capacity < current_capacity) {
877 // Shrinking
878 if (total_full_collections() < AdaptiveSizePolicyReadyThreshold) {
879 // No enough data for shrinking
880 return;
881 }
882 }
883 }
884
885 _old_gen->resize(desired_capacity);
886 }
887
888 void ParallelScavengeHeap::resize_after_young_gc(bool is_survivor_overflowing) {
889 _young_gen->resize_after_young_gc(is_survivor_overflowing);
890
891 // Consider if should shrink old-gen
892 if (!is_survivor_overflowing) {
893 assert(old_gen()->capacity_in_bytes() >= old_gen()->min_gen_size(), "inv");
894
895 // Old gen min_gen_size constraint.
896 const size_t max_shrink_bytes_gen_size_constraint = old_gen()->capacity_in_bytes() - old_gen()->min_gen_size();
897
898 // Per-step delta to avoid too aggressive shrinking.
899 const size_t max_shrink_bytes_per_step_constraint = SpaceAlignment;
900
901 // Combining the above two constraints.
902 const size_t max_shrink_bytes = MIN2(max_shrink_bytes_gen_size_constraint,
903 max_shrink_bytes_per_step_constraint);
904
905 size_t shrink_bytes = _size_policy->compute_old_gen_shrink_bytes(old_gen()->free_in_bytes(), max_shrink_bytes);
906
907 assert(old_gen()->capacity_in_bytes() >= shrink_bytes, "inv");
908 assert(old_gen()->capacity_in_bytes() - shrink_bytes >= old_gen()->min_gen_size(), "inv");
909
910 if (shrink_bytes != 0) {
911 if (MinHeapFreeRatio != 0) {
912 size_t new_capacity = old_gen()->capacity_in_bytes() - shrink_bytes;
913 size_t new_free_size = old_gen()->free_in_bytes() - shrink_bytes;
914 if ((double)new_free_size / new_capacity * 100 < MinHeapFreeRatio) {
915 // Would violate MinHeapFreeRatio
916 return;
917 }
918 }
919 old_gen()->shrink(shrink_bytes);
920 }
921 }
922 }
923
924 void ParallelScavengeHeap::resize_after_full_gc() {
925 resize_old_gen_after_full_gc();
926 // We don't resize young-gen after full-gc because:
927 // 1. eden-size directly affects young-gc frequency (GCTimeRatio), and we
928 // don't have enough info to determine its desired size.
929 // 2. eden can contain live objs after a full-gc, which is unsafe for
930 // resizing. We will perform expansion on allocation if needed, in
931 // satisfy_failed_allocation().
932 }
933
934 HeapWord* ParallelScavengeHeap::allocate_loaded_archive_space(size_t size) {
935 return _old_gen->allocate(size);
936 }
937
938 void ParallelScavengeHeap::complete_loaded_archive_space(MemRegion archive_space) {
939 assert(_old_gen->object_space()->used_region().contains(archive_space),
940 "Archive space not contained in old gen");
941 _old_gen->complete_loaded_archive_space(archive_space);
942 }
943
944 void ParallelScavengeHeap::register_nmethod(nmethod* nm) {
945 ScavengableNMethods::register_nmethod(nm);
946 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
947 bs_nm->disarm(nm);
948 }
949
950 void ParallelScavengeHeap::unregister_nmethod(nmethod* nm) {
951 ScavengableNMethods::unregister_nmethod(nm);
952 }
953
954 void ParallelScavengeHeap::verify_nmethod(nmethod* nm) {
955 ScavengableNMethods::verify_nmethod(nm);
956 }
957
958 GrowableArray<GCMemoryManager*> ParallelScavengeHeap::memory_managers() {
959 GrowableArray<GCMemoryManager*> memory_managers(2);
960 memory_managers.append(_young_manager);
961 memory_managers.append(_old_manager);
962 return memory_managers;
963 }
964
965 GrowableArray<MemoryPool*> ParallelScavengeHeap::memory_pools() {
966 GrowableArray<MemoryPool*> memory_pools(3);
967 memory_pools.append(_eden_pool);
968 memory_pools.append(_survivor_pool);
969 memory_pools.append(_old_pool);
970 return memory_pools;
971 }
972
973 void ParallelScavengeHeap::pin_object(JavaThread* thread, oop obj) {
974 GCLocker::enter(thread);
975 }
976
977 void ParallelScavengeHeap::unpin_object(JavaThread* thread, oop obj) {
978 GCLocker::exit(thread);
979 }
980
981 void ParallelScavengeHeap::update_parallel_worker_threads_cpu_time() {
982 assert(Thread::current()->is_VM_thread(),
983 "Must be called from VM thread to avoid races");
984 if (!UsePerfData || !os::is_thread_cpu_time_supported()) {
985 return;
986 }
987
988 // Ensure ThreadTotalCPUTimeClosure destructor is called before publishing gc
989 // time.
990 {
991 ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
992 // Currently parallel worker threads in GCTaskManager never terminate, so it
993 // is safe for VMThread to read their CPU times. If upstream changes this
994 // behavior, we should rethink if it is still safe.
995 gc_threads_do(&tttc);
996 }
997
998 CPUTimeCounters::publish_gc_total_cpu_time();
999 }