1 /*
2 * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "gc/parallel/objectStartArray.inline.hpp"
26 #include "gc/parallel/parallelArguments.hpp"
27 #include "gc/parallel/parallelInitLogger.hpp"
28 #include "gc/parallel/parallelScavengeHeap.inline.hpp"
29 #include "gc/parallel/psAdaptiveSizePolicy.hpp"
30 #include "gc/parallel/psMemoryPool.hpp"
31 #include "gc/parallel/psParallelCompact.inline.hpp"
32 #include "gc/parallel/psParallelCompactNew.inline.hpp"
33 #include "gc/parallel/psPromotionManager.hpp"
34 #include "gc/parallel/psScavenge.hpp"
35 #include "gc/parallel/psVMOperations.hpp"
36 #include "gc/shared/barrierSetNMethod.hpp"
37 #include "gc/shared/fullGCForwarding.inline.hpp"
38 #include "gc/shared/gcHeapSummary.hpp"
39 #include "gc/shared/gcLocker.inline.hpp"
40 #include "gc/shared/gcWhen.hpp"
41 #include "gc/shared/genArguments.hpp"
42 #include "gc/shared/locationPrinter.inline.hpp"
43 #include "gc/shared/scavengableNMethods.hpp"
44 #include "gc/shared/suspendibleThreadSet.hpp"
45 #include "logging/log.hpp"
46 #include "memory/iterator.hpp"
47 #include "memory/metaspaceCounters.hpp"
48 #include "memory/metaspaceUtils.hpp"
49 #include "memory/reservedSpace.hpp"
50 #include "memory/universe.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "runtime/cpuTimeCounters.hpp"
53 #include "runtime/globals_extension.hpp"
54 #include "runtime/handles.inline.hpp"
55 #include "runtime/init.hpp"
56 #include "runtime/java.hpp"
57 #include "runtime/vmThread.hpp"
58 #include "services/memoryManager.hpp"
59 #include "utilities/macros.hpp"
60 #include "utilities/vmError.hpp"
61
62 PSYoungGen* ParallelScavengeHeap::_young_gen = nullptr;
63 PSOldGen* ParallelScavengeHeap::_old_gen = nullptr;
64 PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = nullptr;
65 GCPolicyCounters* ParallelScavengeHeap::_gc_policy_counters = nullptr;
66 size_t ParallelScavengeHeap::_desired_page_size = 0;
67
68 jint ParallelScavengeHeap::initialize() {
69 const size_t reserved_heap_size = ParallelArguments::heap_reserved_size_bytes();
70
71 assert(_desired_page_size != 0, "Should be initialized");
72 ReservedHeapSpace heap_rs = Universe::reserve_heap(reserved_heap_size, HeapAlignment, _desired_page_size);
73 // Adjust SpaceAlignment based on actually used large page size.
74 if (UseLargePages) {
75 SpaceAlignment = MAX2(heap_rs.page_size(), default_space_alignment());
76 }
77 assert(is_aligned(SpaceAlignment, heap_rs.page_size()), "inv");
78
79 trace_actual_reserved_page_size(reserved_heap_size, heap_rs);
80
81 initialize_reserved_region(heap_rs);
82 // Layout the reserved space for the generations.
83 ReservedSpace old_rs = heap_rs.first_part(MaxOldSize, SpaceAlignment);
84 ReservedSpace young_rs = heap_rs.last_part(MaxOldSize, SpaceAlignment);
85 assert(young_rs.size() == MaxNewSize, "Didn't reserve all of the heap");
86
87 PSCardTable* card_table = new PSCardTable(_reserved);
88 card_table->initialize(old_rs.base(), young_rs.base());
89
90 CardTableBarrierSet* const barrier_set = new CardTableBarrierSet(card_table);
91 BarrierSet::set_barrier_set(barrier_set);
92
93 // Set up WorkerThreads
94 _workers.initialize_workers();
95
96 // Create and initialize the generations.
97 _young_gen = new PSYoungGen(
98 young_rs,
99 NewSize,
100 MinNewSize,
101 MaxNewSize);
102 _old_gen = new PSOldGen(
103 old_rs,
104 OldSize,
105 MinOldSize,
106 MaxOldSize);
107
108 assert(young_gen()->max_gen_size() == young_rs.size(),"Consistency check");
109 assert(old_gen()->max_gen_size() == old_rs.size(), "Consistency check");
110
111 double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0;
112
113 _size_policy = new PSAdaptiveSizePolicy(SpaceAlignment,
114 max_gc_pause_sec);
115
116 assert((old_gen()->virtual_space()->high_boundary() ==
117 young_gen()->virtual_space()->low_boundary()),
118 "Boundaries must meet");
119 // initialize the policy counters - 2 collectors, 2 generations
120 _gc_policy_counters = new GCPolicyCounters("ParScav:MSC", 2, 2);
121
122 if (UseCompactObjectHeaders) {
123 if (!PSParallelCompactNew::initialize_aux_data()) {
124 return JNI_ENOMEM;
125 }
126 } else {
127 if (!PSParallelCompact::initialize_aux_data()) {
128 return JNI_ENOMEM;
129 }
130 }
131
132 // Create CPU time counter
133 CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
134
135 ParallelInitLogger::print();
136
137 FullGCForwarding::initialize(_reserved);
138
139 return JNI_OK;
140 }
141
142 void ParallelScavengeHeap::initialize_serviceability() {
143
144 _eden_pool = new EdenMutableSpacePool(_young_gen,
145 _young_gen->eden_space(),
146 "PS Eden Space",
147 false /* support_usage_threshold */);
148
149 _survivor_pool = new SurvivorMutableSpacePool(_young_gen,
150 "PS Survivor Space",
151 false /* support_usage_threshold */);
152
153 _old_pool = new PSGenerationPool(_old_gen,
154 "PS Old Gen",
155 true /* support_usage_threshold */);
156
157 _young_manager = new GCMemoryManager("PS Scavenge");
158 _old_manager = new GCMemoryManager("PS MarkSweep");
159
160 _old_manager->add_pool(_eden_pool);
161 _old_manager->add_pool(_survivor_pool);
162 _old_manager->add_pool(_old_pool);
163
164 _young_manager->add_pool(_eden_pool);
165 _young_manager->add_pool(_survivor_pool);
166
167 }
168
169 class PSIsScavengable : public BoolObjectClosure {
170 bool do_object_b(oop obj) {
171 return ParallelScavengeHeap::heap()->is_in_young(obj);
172 }
173 };
174
175 static PSIsScavengable _is_scavengable;
176
177 void ParallelScavengeHeap::post_initialize() {
178 CollectedHeap::post_initialize();
179 // Need to init the tenuring threshold
180 PSScavenge::initialize();
181 if (UseCompactObjectHeaders) {
182 PSParallelCompactNew::post_initialize();
183 } else {
184 PSParallelCompact::post_initialize();
185 }
186 PSPromotionManager::initialize();
187
188 ScavengableNMethods::initialize(&_is_scavengable);
189 GCLocker::initialize();
190 }
191
192 void ParallelScavengeHeap::gc_epilogue(bool full) {
193 if (_is_heap_almost_full) {
194 // Reset emergency state if eden is empty after a young/full gc
195 if (_young_gen->eden_space()->is_empty()) {
196 log_debug(gc)("Leaving memory constrained state; back to normal");
197 _is_heap_almost_full = false;
198 }
199 } else {
200 if (full && !_young_gen->eden_space()->is_empty()) {
201 log_debug(gc)("Non-empty young-gen after full-gc; in memory constrained state");
202 _is_heap_almost_full = true;
203 }
204 }
205 }
206
207 void ParallelScavengeHeap::update_counters() {
208 young_gen()->update_counters();
209 old_gen()->update_counters();
210 MetaspaceCounters::update_performance_counters();
211 update_parallel_worker_threads_cpu_time();
212 }
213
214 size_t ParallelScavengeHeap::capacity() const {
215 size_t value = young_gen()->capacity_in_bytes() + old_gen()->capacity_in_bytes();
216 return value;
217 }
218
219 size_t ParallelScavengeHeap::used() const {
220 size_t value = young_gen()->used_in_bytes() + old_gen()->used_in_bytes();
221 return value;
222 }
223
224 size_t ParallelScavengeHeap::max_capacity() const {
225 size_t estimated = reserved_region().byte_size();
226 if (UseAdaptiveSizePolicy) {
227 estimated -= _size_policy->max_survivor_size(young_gen()->max_gen_size());
228 } else {
229 estimated -= young_gen()->to_space()->capacity_in_bytes();
230 }
231 return MAX2(estimated, capacity());
232 }
233
234 bool ParallelScavengeHeap::is_in(const void* p) const {
235 return young_gen()->is_in(p) || old_gen()->is_in(p);
236 }
237
238 bool ParallelScavengeHeap::is_in_reserved(const void* p) const {
239 return young_gen()->is_in_reserved(p) || old_gen()->is_in_reserved(p);
240 }
241
242 bool ParallelScavengeHeap::requires_barriers(stackChunkOop p) const {
243 return !is_in_young(p);
244 }
245
246 // There are two levels of allocation policy here.
247 //
248 // When an allocation request fails, the requesting thread must invoke a VM
249 // operation, transfer control to the VM thread, and await the results of a
250 // garbage collection. That is quite expensive, and we should avoid doing it
251 // multiple times if possible.
252 //
253 // To accomplish this, we have a basic allocation policy, and also a
254 // failed allocation policy.
255 //
256 // The basic allocation policy controls how you allocate memory without
257 // attempting garbage collection. It is okay to grab locks and
258 // expand the heap, if that can be done without coming to a safepoint.
259 // It is likely that the basic allocation policy will not be very
260 // aggressive.
261 //
262 // The failed allocation policy is invoked from the VM thread after
263 // the basic allocation policy is unable to satisfy a mem_allocate
264 // request. This policy needs to cover the entire range of collection,
265 // heap expansion, and out-of-memory conditions. It should make every
266 // attempt to allocate the requested memory.
267
268 // Basic allocation policy. Should never be called at a safepoint, or
269 // from the VM thread.
270 //
271 // This method must handle cases where many mem_allocate requests fail
272 // simultaneously. When that happens, only one VM operation will succeed,
273 // and the rest will not be executed. For that reason, this method loops
274 // during failed allocation attempts. If the java heap becomes exhausted,
275 // we rely on the size_policy object to force a bail out.
276 HeapWord* ParallelScavengeHeap::mem_allocate(size_t size) {
277 assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint");
278 assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread");
279 assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");
280
281 bool is_tlab = false;
282 return mem_allocate_work(size, is_tlab);
283 }
284
285 HeapWord* ParallelScavengeHeap::mem_allocate_cas_noexpand(size_t size, bool is_tlab) {
286 // Try young-gen first.
287 HeapWord* result = young_gen()->allocate(size);
288 if (result != nullptr) {
289 return result;
290 }
291
292 // Try allocating from the old gen for non-TLAB and large allocations.
293 if (!is_tlab) {
294 if (!should_alloc_in_eden(size)) {
295 result = old_gen()->cas_allocate_noexpand(size);
296 if (result != nullptr) {
297 return result;
298 }
299 }
300 }
301
302 // In extreme cases, try allocating in from space also.
303 if (_is_heap_almost_full) {
304 result = young_gen()->from_space()->cas_allocate(size);
305 if (result != nullptr) {
306 return result;
307 }
308 if (!is_tlab) {
309 result = old_gen()->cas_allocate_noexpand(size);
310 if (result != nullptr) {
311 return result;
312 }
313 }
314 }
315
316 return nullptr;
317 }
318
319 HeapWord* ParallelScavengeHeap::mem_allocate_work(size_t size, bool is_tlab) {
320 for (uint loop_count = 0; /* empty */; ++loop_count) {
321 HeapWord* result = mem_allocate_cas_noexpand(size, is_tlab);
322 if (result != nullptr) {
323 return result;
324 }
325
326 // Read total_collections() under the lock so that multiple
327 // allocation-failures result in one GC.
328 uint gc_count;
329 {
330 MutexLocker ml(Heap_lock);
331
332 // Re-try after acquiring the lock, because a GC might have occurred
333 // while waiting for this lock.
334 result = mem_allocate_cas_noexpand(size, is_tlab);
335 if (result != nullptr) {
336 return result;
337 }
338
339 if (!is_init_completed()) {
340 // Can't do GC; try heap expansion to satisfy the request.
341 result = expand_heap_and_allocate(size, is_tlab);
342 if (result != nullptr) {
343 return result;
344 }
345 }
346
347 gc_count = total_collections();
348 }
349
350 {
351 VM_ParallelCollectForAllocation op(size, is_tlab, gc_count);
352 VMThread::execute(&op);
353
354 if (op.gc_succeeded()) {
355 assert(is_in_or_null(op.result()), "result not in heap");
356 return op.result();
357 }
358 }
359
360 // Was the gc-overhead reached inside the safepoint? If so, this mutator
361 // should return null as well for global consistency.
362 if (_gc_overhead_counter >= GCOverheadLimitThreshold) {
363 return nullptr;
364 }
365
366 if ((QueuedAllocationWarningCount > 0) &&
367 (loop_count % QueuedAllocationWarningCount == 0)) {
368 log_warning(gc)("ParallelScavengeHeap::mem_allocate retries %d times, size=%zu", loop_count, size);
369 }
370 }
371 }
372
373 void ParallelScavengeHeap::do_full_collection(bool clear_all_soft_refs) {
374 // No need for max-compaction in this context.
375 const bool should_do_max_compaction = false;
376 if (UseCompactObjectHeaders) {
377 PSParallelCompactNew::invoke(clear_all_soft_refs, should_do_max_compaction);
378 } else {
379 PSParallelCompact::invoke(clear_all_soft_refs, should_do_max_compaction);
380 }
381 }
382
383 bool ParallelScavengeHeap::should_attempt_young_gc() const {
384 const bool ShouldRunYoungGC = true;
385 const bool ShouldRunFullGC = false;
386
387 if (!_young_gen->to_space()->is_empty()) {
388 log_debug(gc, ergo)("To-space is not empty; run full-gc instead.");
389 return ShouldRunFullGC;
390 }
391
392 // Check if the predicted promoted bytes will overflow free space in old-gen.
393 PSAdaptiveSizePolicy* policy = _size_policy;
394
395 size_t avg_promoted = (size_t) policy->padded_average_promoted_in_bytes();
396 size_t promotion_estimate = MIN2(avg_promoted, _young_gen->used_in_bytes());
397 // Total free size after possible old gen expansion
398 size_t free_in_old_gen_with_expansion = _old_gen->max_gen_size() - _old_gen->used_in_bytes();
399
400 log_trace(gc, ergo)("average_promoted %zu; padded_average_promoted %zu",
401 (size_t) policy->average_promoted_in_bytes(),
402 (size_t) policy->padded_average_promoted_in_bytes());
403
404 if (promotion_estimate >= free_in_old_gen_with_expansion) {
405 log_debug(gc, ergo)("Run full-gc; predicted promotion size >= max free space in old-gen: %zu >= %zu",
406 promotion_estimate, free_in_old_gen_with_expansion);
407 return ShouldRunFullGC;
408 }
409
410 if (UseAdaptiveSizePolicy) {
411 // Also checking OS has enough free memory to commit and expand old-gen.
412 // Otherwise, the recorded gc-pause-time might be inflated to include time
413 // of OS preparing free memory, resulting in inaccurate young-gen resizing.
414 assert(_old_gen->committed().byte_size() >= _old_gen->used_in_bytes(), "inv");
415 // Use uint64_t instead of size_t for 32bit compatibility.
416 uint64_t free_mem_in_os;
417 if (os::free_memory(free_mem_in_os)) {
418 size_t actual_free = (size_t)MIN2(_old_gen->committed().byte_size() - _old_gen->used_in_bytes() + free_mem_in_os,
419 (uint64_t)SIZE_MAX);
420 if (promotion_estimate > actual_free) {
421 log_debug(gc, ergo)("Run full-gc; predicted promotion size > free space in old-gen and OS: %zu > %zu",
422 promotion_estimate, actual_free);
423 return ShouldRunFullGC;
424 }
425 }
426 }
427
428 // No particular reasons to run full-gc, so young-gc.
429 return ShouldRunYoungGC;
430 }
431
432 static bool check_gc_heap_free_limit(size_t free_bytes, size_t capacity_bytes) {
433 return (free_bytes * 100 / capacity_bytes) < GCHeapFreeLimit;
434 }
435
436 bool ParallelScavengeHeap::check_gc_overhead_limit() {
437 assert(SafepointSynchronize::is_at_safepoint(), "precondition");
438
439 if (UseGCOverheadLimit) {
440 // The goal here is to return null prematurely so that apps can exit
441 // gracefully when GC takes the most time.
442 bool little_mutator_time = _size_policy->mutator_time_percent() * 100 < (100 - GCTimeLimit);
443 bool little_free_space = check_gc_heap_free_limit(_young_gen->free_in_bytes(), _young_gen->capacity_in_bytes())
444 && check_gc_heap_free_limit( _old_gen->free_in_bytes(), _old_gen->capacity_in_bytes());
445
446 log_debug(gc)("GC Overhead Limit: GC Time %f Free Space Young %f Old %f Counter %zu",
447 (100 - _size_policy->mutator_time_percent()),
448 percent_of(_young_gen->free_in_bytes(), _young_gen->capacity_in_bytes()),
449 percent_of(_old_gen->free_in_bytes(), _young_gen->capacity_in_bytes()),
450 _gc_overhead_counter);
451
452 if (little_mutator_time && little_free_space) {
453 _gc_overhead_counter++;
454 if (_gc_overhead_counter >= GCOverheadLimitThreshold) {
455 return true;
456 }
457 } else {
458 _gc_overhead_counter = 0;
459 }
460 }
461 return false;
462 }
463
464 HeapWord* ParallelScavengeHeap::expand_heap_and_allocate(size_t size, bool is_tlab) {
465 #ifdef ASSERT
466 assert(Heap_lock->is_locked(), "precondition");
467 if (is_init_completed()) {
468 assert(SafepointSynchronize::is_at_safepoint(), "precondition");
469 assert(Thread::current()->is_VM_thread(), "precondition");
470 } else {
471 assert(Thread::current()->is_Java_thread(), "precondition");
472 assert(Heap_lock->owned_by_self(), "precondition");
473 }
474 #endif
475
476 HeapWord* result = young_gen()->expand_and_allocate(size);
477
478 if (result == nullptr && !is_tlab) {
479 result = old_gen()->expand_and_allocate(size);
480 }
481
482 return result; // Could be null if we are out of space.
483 }
484
485 HeapWord* ParallelScavengeHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {
486 assert(size != 0, "precondition");
487
488 HeapWord* result = nullptr;
489
490 if (!_is_heap_almost_full) {
491 // If young-gen can handle this allocation, attempt young-gc firstly, as young-gc is usually cheaper.
492 bool should_run_young_gc = is_tlab || should_alloc_in_eden(size);
493
494 collect_at_safepoint(!should_run_young_gc);
495
496 // If gc-overhead is reached, we will skip allocation.
497 if (!check_gc_overhead_limit()) {
498 result = expand_heap_and_allocate(size, is_tlab);
499 if (result != nullptr) {
500 return result;
501 }
502 }
503 }
504
505 // Last resort GC; clear soft refs and do max-compaction before throwing OOM.
506 {
507 const bool clear_all_soft_refs = true;
508 const bool should_do_max_compaction = true;
509 if (UseCompactObjectHeaders) {
510 PSParallelCompactNew::invoke(clear_all_soft_refs, should_do_max_compaction);
511 } else {
512 PSParallelCompact::invoke(clear_all_soft_refs, should_do_max_compaction);
513 }
514 }
515
516 if (check_gc_overhead_limit()) {
517 log_info(gc)("GC Overhead Limit exceeded too often (%zu).", GCOverheadLimitThreshold);
518 return nullptr;
519 }
520
521 result = expand_heap_and_allocate(size, is_tlab);
522 return result;
523 }
524
525 void ParallelScavengeHeap::ensure_parsability(bool retire_tlabs) {
526 CollectedHeap::ensure_parsability(retire_tlabs);
527 young_gen()->eden_space()->ensure_parsability();
528 }
529
530 size_t ParallelScavengeHeap::tlab_capacity() const {
531 return young_gen()->eden_space()->tlab_capacity();
532 }
533
534 size_t ParallelScavengeHeap::tlab_used() const {
535 return young_gen()->eden_space()->tlab_used();
536 }
537
538 size_t ParallelScavengeHeap::unsafe_max_tlab_alloc() const {
539 return young_gen()->eden_space()->unsafe_max_tlab_alloc();
540 }
541
542 HeapWord* ParallelScavengeHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) {
543 HeapWord* result = mem_allocate_work(requested_size /* size */,
544 true /* is_tlab */);
545 if (result != nullptr) {
546 *actual_size = requested_size;
547 }
548
549 return result;
550 }
551
552 void ParallelScavengeHeap::resize_all_tlabs() {
553 CollectedHeap::resize_all_tlabs();
554 }
555
556 void ParallelScavengeHeap::prune_scavengable_nmethods() {
557 ScavengableNMethods::prune_nmethods_not_into_young();
558 }
559
560 void ParallelScavengeHeap::prune_unlinked_nmethods() {
561 ScavengableNMethods::prune_unlinked_nmethods();
562 }
563
564 void ParallelScavengeHeap::collect(GCCause::Cause cause) {
565 assert(!Heap_lock->owned_by_self(),
566 "this thread should not own the Heap_lock");
567
568 uint gc_count = 0;
569 uint full_gc_count = 0;
570 {
571 MutexLocker ml(Heap_lock);
572 // This value is guarded by the Heap_lock
573 gc_count = total_collections();
574 full_gc_count = total_full_collections();
575 }
576
577 VM_ParallelGCCollect op(gc_count, full_gc_count, cause);
578 VMThread::execute(&op);
579 }
580
581 void ParallelScavengeHeap::collect_at_safepoint(bool is_full) {
582 assert(!GCLocker::is_active(), "precondition");
583 bool clear_soft_refs = GCCause::should_clear_all_soft_refs(_gc_cause);
584
585 if (!is_full && should_attempt_young_gc()) {
586 bool young_gc_success = PSScavenge::invoke(clear_soft_refs);
587 if (young_gc_success) {
588 return;
589 }
590 log_debug(gc, heap)("Upgrade to Full-GC since Young-gc failed.");
591 }
592
593 const bool should_do_max_compaction = false;
594 if (UseCompactObjectHeaders) {
595 PSParallelCompactNew::invoke(clear_soft_refs, should_do_max_compaction);
596 } else {
597 PSParallelCompact::invoke(clear_soft_refs, should_do_max_compaction);
598 }
599 }
600
601 void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) {
602 young_gen()->object_iterate(cl);
603 old_gen()->object_iterate(cl);
604 }
605
606 // The HeapBlockClaimer is used during parallel iteration over the heap,
607 // allowing workers to claim heap areas ("blocks"), gaining exclusive rights to these.
608 // The eden and survivor spaces are treated as single blocks as it is hard to divide
609 // these spaces.
610 // The old space is divided into fixed-size blocks.
611 class HeapBlockClaimer : public StackObj {
612 size_t _claimed_index;
613
614 public:
615 static const size_t InvalidIndex = SIZE_MAX;
616 static const size_t EdenIndex = 0;
617 static const size_t SurvivorIndex = 1;
618 static const size_t NumNonOldGenClaims = 2;
619
620 HeapBlockClaimer() : _claimed_index(EdenIndex) { }
621 // Claim the block and get the block index.
622 size_t claim_and_get_block() {
623 size_t block_index;
624 block_index = AtomicAccess::fetch_then_add(&_claimed_index, 1u);
625
626 PSOldGen* old_gen = ParallelScavengeHeap::heap()->old_gen();
627 size_t num_claims = old_gen->num_iterable_blocks() + NumNonOldGenClaims;
628
629 return block_index < num_claims ? block_index : InvalidIndex;
630 }
631 };
632
633 void ParallelScavengeHeap::object_iterate_parallel(ObjectClosure* cl,
634 HeapBlockClaimer* claimer) {
635 size_t block_index = claimer->claim_and_get_block();
636 // Iterate until all blocks are claimed
637 if (block_index == HeapBlockClaimer::EdenIndex) {
638 young_gen()->eden_space()->object_iterate(cl);
639 block_index = claimer->claim_and_get_block();
640 }
641 if (block_index == HeapBlockClaimer::SurvivorIndex) {
642 young_gen()->from_space()->object_iterate(cl);
643 young_gen()->to_space()->object_iterate(cl);
644 block_index = claimer->claim_and_get_block();
645 }
646 while (block_index != HeapBlockClaimer::InvalidIndex) {
647 old_gen()->object_iterate_block(cl, block_index - HeapBlockClaimer::NumNonOldGenClaims);
648 block_index = claimer->claim_and_get_block();
649 }
650 }
651
652 class PSScavengeParallelObjectIterator : public ParallelObjectIteratorImpl {
653 private:
654 ParallelScavengeHeap* _heap;
655 HeapBlockClaimer _claimer;
656
657 public:
658 PSScavengeParallelObjectIterator() :
659 _heap(ParallelScavengeHeap::heap()),
660 _claimer() {}
661
662 virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
663 _heap->object_iterate_parallel(cl, &_claimer);
664 }
665 };
666
667 ParallelObjectIteratorImpl* ParallelScavengeHeap::parallel_object_iterator(uint thread_num) {
668 return new PSScavengeParallelObjectIterator();
669 }
670
671 HeapWord* ParallelScavengeHeap::block_start(const void* addr) const {
672 if (young_gen()->is_in_reserved(addr)) {
673 assert(young_gen()->is_in(addr),
674 "addr should be in allocated part of young gen");
675 // called from os::print_location by find or VMError
676 if (DebuggingContext::is_enabled() || VMError::is_error_reported()) {
677 return nullptr;
678 }
679 Unimplemented();
680 } else if (old_gen()->is_in_reserved(addr)) {
681 assert(old_gen()->is_in(addr),
682 "addr should be in allocated part of old gen");
683 return old_gen()->start_array()->object_start((HeapWord*)addr);
684 }
685 return nullptr;
686 }
687
688 bool ParallelScavengeHeap::block_is_obj(const HeapWord* addr) const {
689 return block_start(addr) == addr;
690 }
691
692 void ParallelScavengeHeap::prepare_for_verify() {
693 ensure_parsability(false); // no need to retire TLABs for verification
694 }
695
696 PSHeapSummary ParallelScavengeHeap::create_ps_heap_summary() {
697 PSOldGen* old = old_gen();
698 HeapWord* old_committed_end = (HeapWord*)old->virtual_space()->committed_high_addr();
699 HeapWord* old_reserved_start = old->reserved().start();
700 HeapWord* old_reserved_end = old->reserved().end();
701 VirtualSpaceSummary old_summary(old_reserved_start, old_committed_end, old_reserved_end);
702 SpaceSummary old_space(old_reserved_start, old_committed_end, old->used_in_bytes());
703
704 PSYoungGen* young = young_gen();
705 VirtualSpaceSummary young_summary(young->reserved().start(),
706 (HeapWord*)young->virtual_space()->committed_high_addr(), young->reserved().end());
707
708 MutableSpace* eden = young_gen()->eden_space();
709 SpaceSummary eden_space(eden->bottom(), eden->end(), eden->used_in_bytes());
710
711 MutableSpace* from = young_gen()->from_space();
712 SpaceSummary from_space(from->bottom(), from->end(), from->used_in_bytes());
713
714 MutableSpace* to = young_gen()->to_space();
715 SpaceSummary to_space(to->bottom(), to->end(), to->used_in_bytes());
716
717 VirtualSpaceSummary heap_summary = create_heap_space_summary();
718 return PSHeapSummary(heap_summary, used(), old_summary, old_space, young_summary, eden_space, from_space, to_space);
719 }
720
721 bool ParallelScavengeHeap::print_location(outputStream* st, void* addr) const {
722 return BlockLocationPrinter<ParallelScavengeHeap>::print_location(st, addr);
723 }
724
725 void ParallelScavengeHeap::print_heap_on(outputStream* st) const {
726 if (young_gen() != nullptr) {
727 young_gen()->print_on(st);
728 }
729 if (old_gen() != nullptr) {
730 old_gen()->print_on(st);
731 }
732 }
733
734 void ParallelScavengeHeap::print_gc_on(outputStream* st) const {
735 BarrierSet* bs = BarrierSet::barrier_set();
736 if (bs != nullptr) {
737 bs->print_on(st);
738 }
739 st->cr();
740
741 if (UseCompactObjectHeaders) {
742 PSParallelCompactNew::print_on(st);
743 } else {
744 PSParallelCompact::print_on(st);
745 }
746 }
747
748 void ParallelScavengeHeap::gc_threads_do(ThreadClosure* tc) const {
749 ParallelScavengeHeap::heap()->workers().threads_do(tc);
750 }
751
752 void ParallelScavengeHeap::print_tracing_info() const {
753 log_debug(gc, heap, exit)("Accumulated young generation GC time %3.7f secs", PSScavenge::accumulated_time()->seconds());
754 if (UseCompactObjectHeaders) {
755 log_debug(gc, heap, exit)("Accumulated old generation GC time %3.7f secs", PSParallelCompactNew::accumulated_time()->seconds());
756 } else {
757 log_debug(gc, heap, exit)("Accumulated old generation GC time %3.7f secs", PSParallelCompact::accumulated_time()->seconds());
758 }
759 }
760
761 PreGenGCValues ParallelScavengeHeap::get_pre_gc_values() const {
762 const PSYoungGen* const young = young_gen();
763 const MutableSpace* const eden = young->eden_space();
764 const MutableSpace* const from = young->from_space();
765 const PSOldGen* const old = old_gen();
766
767 return PreGenGCValues(young->used_in_bytes(),
768 young->capacity_in_bytes(),
769 eden->used_in_bytes(),
770 eden->capacity_in_bytes(),
771 from->used_in_bytes(),
772 from->capacity_in_bytes(),
773 old->used_in_bytes(),
774 old->capacity_in_bytes());
775 }
776
777 void ParallelScavengeHeap::print_heap_change(const PreGenGCValues& pre_gc_values) const {
778 const PSYoungGen* const young = young_gen();
779 const MutableSpace* const eden = young->eden_space();
780 const MutableSpace* const from = young->from_space();
781 const PSOldGen* const old = old_gen();
782
783 log_info(gc, heap)(HEAP_CHANGE_FORMAT" "
784 HEAP_CHANGE_FORMAT" "
785 HEAP_CHANGE_FORMAT,
786 HEAP_CHANGE_FORMAT_ARGS(young->name(),
787 pre_gc_values.young_gen_used(),
788 pre_gc_values.young_gen_capacity(),
789 young->used_in_bytes(),
790 young->capacity_in_bytes()),
791 HEAP_CHANGE_FORMAT_ARGS("Eden",
792 pre_gc_values.eden_used(),
793 pre_gc_values.eden_capacity(),
794 eden->used_in_bytes(),
795 eden->capacity_in_bytes()),
796 HEAP_CHANGE_FORMAT_ARGS("From",
797 pre_gc_values.from_used(),
798 pre_gc_values.from_capacity(),
799 from->used_in_bytes(),
800 from->capacity_in_bytes()));
801 log_info(gc, heap)(HEAP_CHANGE_FORMAT,
802 HEAP_CHANGE_FORMAT_ARGS(old->name(),
803 pre_gc_values.old_gen_used(),
804 pre_gc_values.old_gen_capacity(),
805 old->used_in_bytes(),
806 old->capacity_in_bytes()));
807 MetaspaceUtils::print_metaspace_change(pre_gc_values.metaspace_sizes());
808 }
809
810 void ParallelScavengeHeap::verify(VerifyOption option /* ignored */) {
811 log_debug(gc, verify)("Tenured");
812 old_gen()->verify();
813
814 log_debug(gc, verify)("Eden");
815 young_gen()->verify();
816
817 log_debug(gc, verify)("CardTable");
818 card_table()->verify_all_young_refs_imprecise();
819 }
820
821 void ParallelScavengeHeap::trace_actual_reserved_page_size(const size_t reserved_heap_size, const ReservedSpace rs) {
822 // Check if Info level is enabled, since os::trace_page_sizes() logs on Info level.
823 if(log_is_enabled(Info, pagesize)) {
824 const size_t page_size = rs.page_size();
825 os::trace_page_sizes("Heap",
826 MinHeapSize,
827 reserved_heap_size,
828 rs.base(),
829 rs.size(),
830 page_size);
831 }
832 }
833
834 void ParallelScavengeHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
835 const PSHeapSummary& heap_summary = create_ps_heap_summary();
836 gc_tracer->report_gc_heap_summary(when, heap_summary);
837
838 const MetaspaceSummary& metaspace_summary = create_metaspace_summary();
839 gc_tracer->report_metaspace_summary(when, metaspace_summary);
840 }
841
842 CardTableBarrierSet* ParallelScavengeHeap::barrier_set() {
843 return barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
844 }
845
846 PSCardTable* ParallelScavengeHeap::card_table() {
847 return static_cast<PSCardTable*>(barrier_set()->card_table());
848 }
849
850 static size_t calculate_free_from_free_ratio_flag(size_t live, uintx free_percent) {
851 assert(free_percent != 100, "precondition");
852 // We want to calculate how much free memory there can be based on the
853 // live size.
854 // percent * (free + live) = free
855 // =>
856 // free = (live * percent) / (1 - percent)
857
858 const double percent = free_percent / 100.0;
859 return live * percent / (1.0 - percent);
860 }
861
862 size_t ParallelScavengeHeap::calculate_desired_old_gen_capacity(size_t old_gen_live_size) {
863 // If min free percent is 100%, the old-gen should always be in its max capacity
864 if (MinHeapFreeRatio == 100) {
865 return _old_gen->max_gen_size();
866 }
867
868 // Using recorded data to calculate the new capacity of old-gen to avoid
869 // excessive expansion but also keep footprint low
870
871 size_t promoted_estimate = _size_policy->padded_average_promoted_in_bytes();
872 // Should have at least this free room for the next young-gc promotion.
873 size_t free_size = promoted_estimate;
874
875 size_t largest_live_size = MAX2((size_t)_size_policy->peak_old_gen_used_estimate(), old_gen_live_size);
876 free_size += largest_live_size - old_gen_live_size;
877
878 // Respect free percent
879 if (MinHeapFreeRatio != 0) {
880 size_t min_free = calculate_free_from_free_ratio_flag(old_gen_live_size, MinHeapFreeRatio);
881 free_size = MAX2(free_size, min_free);
882 }
883
884 if (MaxHeapFreeRatio != 100) {
885 size_t max_free = calculate_free_from_free_ratio_flag(old_gen_live_size, MaxHeapFreeRatio);
886 free_size = MIN2(max_free, free_size);
887 }
888
889 return old_gen_live_size + free_size;
890 }
891
892 void ParallelScavengeHeap::resize_old_gen_after_full_gc() {
893 size_t current_capacity = _old_gen->capacity_in_bytes();
894 size_t desired_capacity = calculate_desired_old_gen_capacity(old_gen()->used_in_bytes());
895
896 // If MinHeapFreeRatio is at its default value; shrink cautiously. Otherwise, users expect prompt shrinking.
897 if (FLAG_IS_DEFAULT(MinHeapFreeRatio)) {
898 if (desired_capacity < current_capacity) {
899 // Shrinking
900 if (total_full_collections() < AdaptiveSizePolicyReadyThreshold) {
901 // No enough data for shrinking
902 return;
903 }
904 }
905 }
906
907 _old_gen->resize(desired_capacity);
908 }
909
910 void ParallelScavengeHeap::resize_after_young_gc(bool is_survivor_overflowing) {
911 _young_gen->resize_after_young_gc(is_survivor_overflowing);
912
913 // Consider if should shrink old-gen
914 if (!is_survivor_overflowing) {
915 // Upper bound for a single step shrink
916 size_t max_shrink_bytes = SpaceAlignment;
917 size_t shrink_bytes = _size_policy->compute_old_gen_shrink_bytes(old_gen()->free_in_bytes(), max_shrink_bytes);
918 if (shrink_bytes != 0) {
919 if (MinHeapFreeRatio != 0) {
920 size_t new_capacity = old_gen()->capacity_in_bytes() - shrink_bytes;
921 size_t new_free_size = old_gen()->free_in_bytes() - shrink_bytes;
922 if ((double)new_free_size / new_capacity * 100 < MinHeapFreeRatio) {
923 // Would violate MinHeapFreeRatio
924 return;
925 }
926 }
927 old_gen()->shrink(shrink_bytes);
928 }
929 }
930 }
931
932 void ParallelScavengeHeap::resize_after_full_gc() {
933 resize_old_gen_after_full_gc();
934 // We don't resize young-gen after full-gc because:
935 // 1. eden-size directly affects young-gc frequency (GCTimeRatio), and we
936 // don't have enough info to determine its desired size.
937 // 2. eden can contain live objs after a full-gc, which is unsafe for
938 // resizing. We will perform expansion on allocation if needed, in
939 // satisfy_failed_allocation().
940 }
941
942 HeapWord* ParallelScavengeHeap::allocate_loaded_archive_space(size_t size) {
943 return _old_gen->allocate(size);
944 }
945
946 void ParallelScavengeHeap::complete_loaded_archive_space(MemRegion archive_space) {
947 assert(_old_gen->object_space()->used_region().contains(archive_space),
948 "Archive space not contained in old gen");
949 _old_gen->complete_loaded_archive_space(archive_space);
950 }
951
952 void ParallelScavengeHeap::register_nmethod(nmethod* nm) {
953 ScavengableNMethods::register_nmethod(nm);
954 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
955 bs_nm->disarm(nm);
956 }
957
958 void ParallelScavengeHeap::unregister_nmethod(nmethod* nm) {
959 ScavengableNMethods::unregister_nmethod(nm);
960 }
961
962 void ParallelScavengeHeap::verify_nmethod(nmethod* nm) {
963 ScavengableNMethods::verify_nmethod(nm);
964 }
965
966 GrowableArray<GCMemoryManager*> ParallelScavengeHeap::memory_managers() {
967 GrowableArray<GCMemoryManager*> memory_managers(2);
968 memory_managers.append(_young_manager);
969 memory_managers.append(_old_manager);
970 return memory_managers;
971 }
972
973 GrowableArray<MemoryPool*> ParallelScavengeHeap::memory_pools() {
974 GrowableArray<MemoryPool*> memory_pools(3);
975 memory_pools.append(_eden_pool);
976 memory_pools.append(_survivor_pool);
977 memory_pools.append(_old_pool);
978 return memory_pools;
979 }
980
981 void ParallelScavengeHeap::pin_object(JavaThread* thread, oop obj) {
982 GCLocker::enter(thread);
983 }
984
985 void ParallelScavengeHeap::unpin_object(JavaThread* thread, oop obj) {
986 GCLocker::exit(thread);
987 }
988
989 void ParallelScavengeHeap::update_parallel_worker_threads_cpu_time() {
990 assert(Thread::current()->is_VM_thread(),
991 "Must be called from VM thread to avoid races");
992 if (!UsePerfData || !os::is_thread_cpu_time_supported()) {
993 return;
994 }
995
996 // Ensure ThreadTotalCPUTimeClosure destructor is called before publishing gc
997 // time.
998 {
999 ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
1000 // Currently parallel worker threads in GCTaskManager never terminate, so it
1001 // is safe for VMThread to read their CPU times. If upstream changes this
1002 // behavior, we should rethink if it is still safe.
1003 gc_threads_do(&tttc);
1004 }
1005
1006 CPUTimeCounters::publish_gc_total_cpu_time();
1007 }