1 /*
2 * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/serial/cardTableRS.hpp"
27 #include "gc/serial/defNewGeneration.inline.hpp"
28 #include "gc/serial/serialGcRefProcProxyTask.hpp"
29 #include "gc/serial/serialHeap.inline.hpp"
30 #include "gc/serial/serialStringDedup.inline.hpp"
31 #include "gc/serial/tenuredGeneration.hpp"
32 #include "gc/shared/adaptiveSizePolicy.hpp"
33 #include "gc/shared/ageTable.inline.hpp"
34 #include "gc/shared/collectorCounters.hpp"
35 #include "gc/shared/continuationGCSupport.inline.hpp"
36 #include "gc/shared/gcArguments.hpp"
37 #include "gc/shared/gcHeapSummary.hpp"
38 #include "gc/shared/gcLocker.hpp"
39 #include "gc/shared/gcPolicyCounters.hpp"
40 #include "gc/shared/gcTimer.hpp"
41 #include "gc/shared/gcTrace.hpp"
42 #include "gc/shared/gcTraceTime.inline.hpp"
43 #include "gc/shared/generationSpec.hpp"
44 #include "gc/shared/preservedMarks.inline.hpp"
45 #include "gc/shared/referencePolicy.hpp"
46 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
47 #include "gc/shared/space.inline.hpp"
48 #include "gc/shared/spaceDecorator.inline.hpp"
49 #include "gc/shared/strongRootsScope.hpp"
50 #include "gc/shared/weakProcessor.hpp"
51 #include "logging/log.hpp"
52 #include "memory/iterator.inline.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/instanceRefKlass.hpp"
55 #include "oops/oop.inline.hpp"
56 #include "runtime/java.hpp"
57 #include "runtime/javaThread.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "runtime/threads.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/copy.hpp"
62 #include "utilities/globalDefinitions.hpp"
63 #include "utilities/stack.inline.hpp"
64
65 class ScavengeHelper {
66 DefNewGeneration* _young_gen;
67 HeapWord* _young_gen_end;
68 public:
69 ScavengeHelper(DefNewGeneration* young_gen) :
70 _young_gen(young_gen),
71 _young_gen_end(young_gen->reserved().end()) {}
72
73 bool is_in_young_gen(void* p) const {
74 return p < _young_gen_end;
75 }
76
77 template <typename T, typename Func>
78 void try_scavenge(T* p, Func&& f) {
79 T heap_oop = RawAccess<>::oop_load(p);
80 // Should we copy the obj?
81 if (!CompressedOops::is_null(heap_oop)) {
82 oop obj = CompressedOops::decode_not_null(heap_oop);
83 if (is_in_young_gen(obj)) {
84 assert(!_young_gen->to()->is_in_reserved(obj), "Scanning field twice?");
85 oop new_obj = obj->is_forwarded() ? obj->forwardee()
86 : _young_gen->copy_to_survivor_space(obj);
87 RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
88
89 // callback
90 f(new_obj);
91 }
92 }
93 }
94 };
95
96 class InHeapScanClosure : public BasicOopIterateClosure {
97 ScavengeHelper _helper;
98 protected:
99 bool is_in_young_gen(void* p) const {
100 return _helper.is_in_young_gen(p);
101 }
102
103 template <typename T, typename Func>
104 void try_scavenge(T* p, Func&& f) {
105 _helper.try_scavenge(p, f);
106 }
107
108 InHeapScanClosure(DefNewGeneration* young_gen) :
109 BasicOopIterateClosure(young_gen->ref_processor()),
110 _helper(young_gen) {}
111 };
112
113 class OffHeapScanClosure : public OopClosure {
114 ScavengeHelper _helper;
115 protected:
116 bool is_in_young_gen(void* p) const {
117 return _helper.is_in_young_gen(p);
118 }
119
120 template <typename T, typename Func>
121 void try_scavenge(T* p, Func&& f) {
122 _helper.try_scavenge(p, f);
123 }
124
125 OffHeapScanClosure(DefNewGeneration* young_gen) : _helper(young_gen) {}
126 };
127
128 class OldGenScanClosure : public InHeapScanClosure {
129 CardTableRS* _rs;
130
131 template <typename T>
132 void do_oop_work(T* p) {
133 assert(!is_in_young_gen(p), "precondition");
134
135 try_scavenge(p, [&] (oop new_obj) {
136 // If p points to a younger generation, mark the card.
137 if (is_in_young_gen(new_obj)) {
138 _rs->inline_write_ref_field_gc(p);
139 }
140 });
141 }
142 public:
143 OldGenScanClosure(DefNewGeneration* g) : InHeapScanClosure(g),
144 _rs(SerialHeap::heap()->rem_set()) {}
145
146 void do_oop(oop* p) { do_oop_work(p); }
147 void do_oop(narrowOop* p) { do_oop_work(p); }
148 };
149
150 class PromoteFailureClosure : public InHeapScanClosure {
151 template <typename T>
152 void do_oop_work(T* p) {
153 assert(is_in_young_gen(p), "promote-fail objs must be in young-gen");
154 assert(!SerialHeap::heap()->young_gen()->to()->is_in_reserved(p), "must not be in to-space");
155
156 try_scavenge(p, [] (auto) {});
157 }
158 public:
159 PromoteFailureClosure(DefNewGeneration* g) : InHeapScanClosure(g) {}
160
161 void do_oop(oop* p) { do_oop_work(p); }
162 void do_oop(narrowOop* p) { do_oop_work(p); }
163 };
164
165 class YoungGenScanClosure : public InHeapScanClosure {
166 template <typename T>
167 void do_oop_work(T* p) {
168 assert(SerialHeap::heap()->young_gen()->to()->is_in_reserved(p), "precondition");
169
170 try_scavenge(p, [] (auto) {});
171 }
172 public:
173 YoungGenScanClosure(DefNewGeneration* g) : InHeapScanClosure(g) {}
174
175 void do_oop(oop* p) { do_oop_work(p); }
176 void do_oop(narrowOop* p) { do_oop_work(p); }
177 };
178
179 class RootScanClosure : public OffHeapScanClosure {
180 template <typename T>
181 void do_oop_work(T* p) {
182 assert(!SerialHeap::heap()->is_in_reserved(p), "outside the heap");
183
184 try_scavenge(p, [] (auto) {});
185 }
186 public:
187 RootScanClosure(DefNewGeneration* g) : OffHeapScanClosure(g) {}
188
189 void do_oop(oop* p) { do_oop_work(p); }
190 void do_oop(narrowOop* p) { do_oop_work(p); }
191 };
192
193 class CLDScanClosure: public CLDClosure {
194
195 class CLDOopClosure : public OffHeapScanClosure {
196 ClassLoaderData* _scanned_cld;
197
198 template <typename T>
199 void do_oop_work(T* p) {
200 assert(!SerialHeap::heap()->is_in_reserved(p), "outside the heap");
201
202 try_scavenge(p, [&] (oop new_obj) {
203 assert(_scanned_cld != nullptr, "inv");
204 if (is_in_young_gen(new_obj) && !_scanned_cld->has_modified_oops()) {
205 _scanned_cld->record_modified_oops();
206 }
207 });
208 }
209
210 public:
211 CLDOopClosure(DefNewGeneration* g) : OffHeapScanClosure(g),
212 _scanned_cld(nullptr) {}
213
214 void set_scanned_cld(ClassLoaderData* cld) {
215 assert(cld == nullptr || _scanned_cld == nullptr, "Must be");
216 _scanned_cld = cld;
217 }
218
219 void do_oop(oop* p) { do_oop_work(p); }
220 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
221 };
222
223 CLDOopClosure _oop_closure;
224 public:
225 CLDScanClosure(DefNewGeneration* g) : _oop_closure(g) {}
226
227 void do_cld(ClassLoaderData* cld) {
228 // If the cld has not been dirtied we know that there's
229 // no references into the young gen and we can skip it.
230 if (cld->has_modified_oops()) {
231
232 // Tell the closure which CLD is being scanned so that it can be dirtied
233 // if oops are left pointing into the young gen.
234 _oop_closure.set_scanned_cld(cld);
235
236 // Clean the cld since we're going to scavenge all the metadata.
237 cld->oops_do(&_oop_closure, ClassLoaderData::_claim_none, /*clear_modified_oops*/true);
238
239 _oop_closure.set_scanned_cld(nullptr);
240 }
241 }
242 };
243
244 class IsAliveClosure: public BoolObjectClosure {
245 HeapWord* _young_gen_end;
246 public:
247 IsAliveClosure(DefNewGeneration* g): _young_gen_end(g->reserved().end()) {}
248
249 bool do_object_b(oop p) {
250 return cast_from_oop<HeapWord*>(p) >= _young_gen_end || p->is_forwarded();
251 }
252 };
253
254 class AdjustWeakRootClosure: public OffHeapScanClosure {
255 template <class T>
256 void do_oop_work(T* p) {
257 DEBUG_ONLY(SerialHeap* heap = SerialHeap::heap();)
258 assert(!heap->is_in_reserved(p), "outside the heap");
259
260 oop obj = RawAccess<IS_NOT_NULL>::oop_load(p);
261 if (is_in_young_gen(obj)) {
262 assert(!heap->young_gen()->to()->is_in_reserved(obj), "inv");
263 assert(obj->is_forwarded(), "forwarded before weak-root-processing");
264 oop new_obj = obj->forwardee();
265 RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
266 }
267 }
268 public:
269 AdjustWeakRootClosure(DefNewGeneration* g): OffHeapScanClosure(g) {}
270
271 void do_oop(oop* p) { do_oop_work(p); }
272 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
273 };
274
275 class KeepAliveClosure: public OopClosure {
276 DefNewGeneration* _young_gen;
277 HeapWord* _young_gen_end;
278 CardTableRS* _rs;
279
280 bool is_in_young_gen(void* p) const {
281 return p < _young_gen_end;
282 }
283
284 template <class T>
285 void do_oop_work(T* p) {
286 oop obj = RawAccess<IS_NOT_NULL>::oop_load(p);
287
288 if (is_in_young_gen(obj)) {
289 oop new_obj = obj->is_forwarded() ? obj->forwardee()
290 : _young_gen->copy_to_survivor_space(obj);
291 RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
292
293 if (is_in_young_gen(new_obj) && !is_in_young_gen(p)) {
294 _rs->inline_write_ref_field_gc(p);
295 }
296 }
297 }
298 public:
299 KeepAliveClosure(DefNewGeneration* g) :
300 _young_gen(g),
301 _young_gen_end(g->reserved().end()),
302 _rs(SerialHeap::heap()->rem_set()) {}
303
304 void do_oop(oop* p) { do_oop_work(p); }
305 void do_oop(narrowOop* p) { do_oop_work(p); }
306 };
307
308 class FastEvacuateFollowersClosure: public VoidClosure {
309 SerialHeap* _heap;
310 YoungGenScanClosure* _young_cl;
311 OldGenScanClosure* _old_cl;
312 public:
313 FastEvacuateFollowersClosure(SerialHeap* heap,
314 YoungGenScanClosure* young_cl,
315 OldGenScanClosure* old_cl) :
316 _heap(heap), _young_cl(young_cl), _old_cl(old_cl)
317 {}
318
319 void do_void() {
320 do {
321 _heap->oop_since_save_marks_iterate(_young_cl, _old_cl);
322 } while (!_heap->no_allocs_since_save_marks());
323 guarantee(_heap->young_gen()->promo_failure_scan_is_complete(), "Failed to finish scan");
324 }
325 };
326
327 DefNewGeneration::DefNewGeneration(ReservedSpace rs,
328 size_t initial_size,
329 size_t min_size,
330 size_t max_size,
331 const char* policy)
332 : Generation(rs, initial_size),
333 _preserved_marks_set(false /* in_c_heap */),
334 _promo_failure_drain_in_progress(false),
335 _should_allocate_from_space(false),
336 _string_dedup_requests()
337 {
338 MemRegion cmr((HeapWord*)_virtual_space.low(),
339 (HeapWord*)_virtual_space.high());
340 GenCollectedHeap* gch = GenCollectedHeap::heap();
341
342 gch->rem_set()->resize_covered_region(cmr);
343
344 _eden_space = new ContiguousSpace();
345 _from_space = new ContiguousSpace();
346 _to_space = new ContiguousSpace();
347
348 // Compute the maximum eden and survivor space sizes. These sizes
349 // are computed assuming the entire reserved space is committed.
350 // These values are exported as performance counters.
351 uintx size = _virtual_space.reserved_size();
352 _max_survivor_size = compute_survivor_size(size, SpaceAlignment);
353 _max_eden_size = size - (2*_max_survivor_size);
354
355 // allocate the performance counters
356
357 // Generation counters -- generation 0, 3 subspaces
358 _gen_counters = new GenerationCounters("new", 0, 3,
359 min_size, max_size, &_virtual_space);
360 _gc_counters = new CollectorCounters(policy, 0);
361
362 _eden_counters = new CSpaceCounters("eden", 0, _max_eden_size, _eden_space,
363 _gen_counters);
364 _from_counters = new CSpaceCounters("s0", 1, _max_survivor_size, _from_space,
365 _gen_counters);
366 _to_counters = new CSpaceCounters("s1", 2, _max_survivor_size, _to_space,
367 _gen_counters);
368
369 compute_space_boundaries(0, SpaceDecorator::Clear, SpaceDecorator::Mangle);
370 update_counters();
371 _old_gen = nullptr;
372 _tenuring_threshold = MaxTenuringThreshold;
373 _pretenure_size_threshold_words = PretenureSizeThreshold >> LogHeapWordSize;
374
375 _gc_timer = new STWGCTimer();
376
377 _gc_tracer = new DefNewTracer();
378 }
379
380 void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size,
381 bool clear_space,
382 bool mangle_space) {
383 // If the spaces are being cleared (only done at heap initialization
384 // currently), the survivor spaces need not be empty.
385 // Otherwise, no care is taken for used areas in the survivor spaces
386 // so check.
387 assert(clear_space || (to()->is_empty() && from()->is_empty()),
388 "Initialization of the survivor spaces assumes these are empty");
389
390 // Compute sizes
391 uintx size = _virtual_space.committed_size();
392 uintx survivor_size = compute_survivor_size(size, SpaceAlignment);
393 uintx eden_size = size - (2*survivor_size);
394 if (eden_size > max_eden_size()) {
395 eden_size = max_eden_size();
396 survivor_size = (size - eden_size)/2;
397 }
398 assert(eden_size > 0 && survivor_size <= eden_size, "just checking");
399
400 if (eden_size < minimum_eden_size) {
401 // May happen due to 64Kb rounding, if so adjust eden size back up
402 minimum_eden_size = align_up(minimum_eden_size, SpaceAlignment);
403 uintx maximum_survivor_size = (size - minimum_eden_size) / 2;
404 uintx unaligned_survivor_size =
405 align_down(maximum_survivor_size, SpaceAlignment);
406 survivor_size = MAX2(unaligned_survivor_size, SpaceAlignment);
407 eden_size = size - (2*survivor_size);
408 assert(eden_size > 0 && survivor_size <= eden_size, "just checking");
409 assert(eden_size >= minimum_eden_size, "just checking");
410 }
411
412 char *eden_start = _virtual_space.low();
413 char *from_start = eden_start + eden_size;
414 char *to_start = from_start + survivor_size;
415 char *to_end = to_start + survivor_size;
416
417 assert(to_end == _virtual_space.high(), "just checking");
418 assert(Space::is_aligned(eden_start), "checking alignment");
419 assert(Space::is_aligned(from_start), "checking alignment");
420 assert(Space::is_aligned(to_start), "checking alignment");
421
422 MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start);
423 MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start);
424 MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end);
425
426 // A minimum eden size implies that there is a part of eden that
427 // is being used and that affects the initialization of any
428 // newly formed eden.
429 bool live_in_eden = minimum_eden_size > 0;
430
431 // If not clearing the spaces, do some checking to verify that
432 // the space are already mangled.
433 if (!clear_space) {
434 // Must check mangling before the spaces are reshaped. Otherwise,
435 // the bottom or end of one space may have moved into another
436 // a failure of the check may not correctly indicate which space
437 // is not properly mangled.
438 if (ZapUnusedHeapArea) {
439 HeapWord* limit = (HeapWord*) _virtual_space.high();
440 eden()->check_mangled_unused_area(limit);
441 from()->check_mangled_unused_area(limit);
442 to()->check_mangled_unused_area(limit);
443 }
444 }
445
446 // Reset the spaces for their new regions.
447 eden()->initialize(edenMR,
448 clear_space && !live_in_eden,
449 SpaceDecorator::Mangle);
450 // If clear_space and live_in_eden, we will not have cleared any
451 // portion of eden above its top. This can cause newly
452 // expanded space not to be mangled if using ZapUnusedHeapArea.
453 // We explicitly do such mangling here.
454 if (ZapUnusedHeapArea && clear_space && live_in_eden && mangle_space) {
455 eden()->mangle_unused_area();
456 }
457 from()->initialize(fromMR, clear_space, mangle_space);
458 to()->initialize(toMR, clear_space, mangle_space);
459
460 // Set next compaction spaces.
461 eden()->set_next_compaction_space(from());
462 // The to-space is normally empty before a compaction so need
463 // not be considered. The exception is during promotion
464 // failure handling when to-space can contain live objects.
465 from()->set_next_compaction_space(nullptr);
466 }
467
468 void DefNewGeneration::swap_spaces() {
469 ContiguousSpace* s = from();
470 _from_space = to();
471 _to_space = s;
472 eden()->set_next_compaction_space(from());
473 // The to-space is normally empty before a compaction so need
474 // not be considered. The exception is during promotion
475 // failure handling when to-space can contain live objects.
476 from()->set_next_compaction_space(nullptr);
477
478 if (UsePerfData) {
479 CSpaceCounters* c = _from_counters;
480 _from_counters = _to_counters;
481 _to_counters = c;
482 }
483 }
484
485 bool DefNewGeneration::expand(size_t bytes) {
486 HeapWord* prev_high = (HeapWord*) _virtual_space.high();
487 bool success = _virtual_space.expand_by(bytes);
488 if (success && ZapUnusedHeapArea) {
489 // Mangle newly committed space immediately because it
490 // can be done here more simply that after the new
491 // spaces have been computed.
492 HeapWord* new_high = (HeapWord*) _virtual_space.high();
493 MemRegion mangle_region(prev_high, new_high);
494 SpaceMangler::mangle_region(mangle_region);
495 }
496
497 // Do not attempt an expand-to-the reserve size. The
498 // request should properly observe the maximum size of
499 // the generation so an expand-to-reserve should be
500 // unnecessary. Also a second call to expand-to-reserve
501 // value potentially can cause an undue expansion.
502 // For example if the first expand fail for unknown reasons,
503 // but the second succeeds and expands the heap to its maximum
504 // value.
505 if (GCLocker::is_active()) {
506 log_debug(gc)("Garbage collection disabled, expanded heap instead");
507 }
508
509 return success;
510 }
511
512 size_t DefNewGeneration::calculate_thread_increase_size(int threads_count) const {
513 size_t thread_increase_size = 0;
514 // Check an overflow at 'threads_count * NewSizeThreadIncrease'.
515 if (threads_count > 0 && NewSizeThreadIncrease <= max_uintx / threads_count) {
516 thread_increase_size = threads_count * NewSizeThreadIncrease;
517 }
518 return thread_increase_size;
519 }
520
521 size_t DefNewGeneration::adjust_for_thread_increase(size_t new_size_candidate,
522 size_t new_size_before,
523 size_t alignment,
524 size_t thread_increase_size) const {
525 size_t desired_new_size = new_size_before;
526
527 if (NewSizeThreadIncrease > 0 && thread_increase_size > 0) {
528
529 // 1. Check an overflow at 'new_size_candidate + thread_increase_size'.
530 if (new_size_candidate <= max_uintx - thread_increase_size) {
531 new_size_candidate += thread_increase_size;
532
533 // 2. Check an overflow at 'align_up'.
534 size_t aligned_max = ((max_uintx - alignment) & ~(alignment-1));
535 if (new_size_candidate <= aligned_max) {
536 desired_new_size = align_up(new_size_candidate, alignment);
537 }
538 }
539 }
540
541 return desired_new_size;
542 }
543
544 void DefNewGeneration::compute_new_size() {
545 // This is called after a GC that includes the old generation, so from-space
546 // will normally be empty.
547 // Note that we check both spaces, since if scavenge failed they revert roles.
548 // If not we bail out (otherwise we would have to relocate the objects).
549 if (!from()->is_empty() || !to()->is_empty()) {
550 return;
551 }
552
553 GenCollectedHeap* gch = GenCollectedHeap::heap();
554
555 size_t old_size = gch->old_gen()->capacity();
556 size_t new_size_before = _virtual_space.committed_size();
557 size_t min_new_size = initial_size();
558 size_t max_new_size = reserved().byte_size();
559 assert(min_new_size <= new_size_before &&
560 new_size_before <= max_new_size,
561 "just checking");
562 // All space sizes must be multiples of Generation::GenGrain.
563 size_t alignment = Generation::GenGrain;
564
565 int threads_count = Threads::number_of_non_daemon_threads();
566 size_t thread_increase_size = calculate_thread_increase_size(threads_count);
567
568 size_t new_size_candidate = old_size / NewRatio;
569 // Compute desired new generation size based on NewRatio and NewSizeThreadIncrease
570 // and reverts to previous value if any overflow happens
571 size_t desired_new_size = adjust_for_thread_increase(new_size_candidate, new_size_before,
572 alignment, thread_increase_size);
573
574 // Adjust new generation size
575 desired_new_size = clamp(desired_new_size, min_new_size, max_new_size);
576 assert(desired_new_size <= max_new_size, "just checking");
577
578 bool changed = false;
579 if (desired_new_size > new_size_before) {
580 size_t change = desired_new_size - new_size_before;
581 assert(change % alignment == 0, "just checking");
582 if (expand(change)) {
583 changed = true;
584 }
585 // If the heap failed to expand to the desired size,
586 // "changed" will be false. If the expansion failed
587 // (and at this point it was expected to succeed),
588 // ignore the failure (leaving "changed" as false).
589 }
590 if (desired_new_size < new_size_before && eden()->is_empty()) {
591 // bail out of shrinking if objects in eden
592 size_t change = new_size_before - desired_new_size;
593 assert(change % alignment == 0, "just checking");
594 _virtual_space.shrink_by(change);
595 changed = true;
596 }
597 if (changed) {
598 // The spaces have already been mangled at this point but
599 // may not have been cleared (set top = bottom) and should be.
600 // Mangling was done when the heap was being expanded.
601 compute_space_boundaries(eden()->used(),
602 SpaceDecorator::Clear,
603 SpaceDecorator::DontMangle);
604 MemRegion cmr((HeapWord*)_virtual_space.low(),
605 (HeapWord*)_virtual_space.high());
606 gch->rem_set()->resize_covered_region(cmr);
607
608 log_debug(gc, ergo, heap)(
609 "New generation size " SIZE_FORMAT "K->" SIZE_FORMAT "K [eden=" SIZE_FORMAT "K,survivor=" SIZE_FORMAT "K]",
610 new_size_before/K, _virtual_space.committed_size()/K,
611 eden()->capacity()/K, from()->capacity()/K);
612 log_trace(gc, ergo, heap)(
613 " [allowed " SIZE_FORMAT "K extra for %d threads]",
614 thread_increase_size/K, threads_count);
615 }
616 }
617
618
619 size_t DefNewGeneration::capacity() const {
620 return eden()->capacity()
621 + from()->capacity(); // to() is only used during scavenge
622 }
623
624
625 size_t DefNewGeneration::used() const {
626 return eden()->used()
627 + from()->used(); // to() is only used during scavenge
628 }
629
630
631 size_t DefNewGeneration::free() const {
632 return eden()->free()
633 + from()->free(); // to() is only used during scavenge
634 }
635
636 size_t DefNewGeneration::max_capacity() const {
637 const size_t reserved_bytes = reserved().byte_size();
638 return reserved_bytes - compute_survivor_size(reserved_bytes, SpaceAlignment);
639 }
640
641 size_t DefNewGeneration::unsafe_max_alloc_nogc() const {
642 return eden()->free();
643 }
644
645 size_t DefNewGeneration::capacity_before_gc() const {
646 return eden()->capacity();
647 }
648
649 size_t DefNewGeneration::contiguous_available() const {
650 return eden()->free();
651 }
652
653
654 void DefNewGeneration::object_iterate(ObjectClosure* blk) {
655 eden()->object_iterate(blk);
656 from()->object_iterate(blk);
657 }
658
659
660 void DefNewGeneration::space_iterate(SpaceClosure* blk,
661 bool usedOnly) {
662 blk->do_space(eden());
663 blk->do_space(from());
664 blk->do_space(to());
665 }
666
667 // The last collection bailed out, we are running out of heap space,
668 // so we try to allocate the from-space, too.
669 HeapWord* DefNewGeneration::allocate_from_space(size_t size) {
670 bool should_try_alloc = should_allocate_from_space() || GCLocker::is_active_and_needs_gc();
671
672 // If the Heap_lock is not locked by this thread, this will be called
673 // again later with the Heap_lock held.
674 bool do_alloc = should_try_alloc && (Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()));
675
676 HeapWord* result = nullptr;
677 if (do_alloc) {
678 result = from()->allocate(size);
679 }
680
681 log_trace(gc, alloc)("DefNewGeneration::allocate_from_space(" SIZE_FORMAT "): will_fail: %s heap_lock: %s free: " SIZE_FORMAT "%s%s returns %s",
682 size,
683 GenCollectedHeap::heap()->incremental_collection_will_fail(false /* don't consult_young */) ?
684 "true" : "false",
685 Heap_lock->is_locked() ? "locked" : "unlocked",
686 from()->free(),
687 should_try_alloc ? "" : " should_allocate_from_space: NOT",
688 do_alloc ? " Heap_lock is not owned by self" : "",
689 result == nullptr ? "null" : "object");
690
691 return result;
692 }
693
694 HeapWord* DefNewGeneration::expand_and_allocate(size_t size, bool is_tlab) {
695 // We don't attempt to expand the young generation (but perhaps we should.)
696 return allocate(size, is_tlab);
697 }
698
699 void DefNewGeneration::adjust_desired_tenuring_threshold() {
700 // Set the desired survivor size to half the real survivor space
701 size_t const survivor_capacity = to()->capacity() / HeapWordSize;
702 size_t const desired_survivor_size = (size_t)((((double)survivor_capacity) * TargetSurvivorRatio) / 100);
703
704 _tenuring_threshold = age_table()->compute_tenuring_threshold(desired_survivor_size);
705
706 if (UsePerfData) {
707 GCPolicyCounters* gc_counters = GenCollectedHeap::heap()->counters();
708 gc_counters->tenuring_threshold()->set_value(_tenuring_threshold);
709 gc_counters->desired_survivor_size()->set_value(desired_survivor_size * oopSize);
710 }
711
712 age_table()->print_age_table(_tenuring_threshold);
713 }
714
715 void DefNewGeneration::collect(bool full,
716 bool clear_all_soft_refs,
717 size_t size,
718 bool is_tlab) {
719 assert(full || size > 0, "otherwise we don't want to collect");
720
721 SerialHeap* heap = SerialHeap::heap();
722
723 _gc_timer->register_gc_start();
724 _gc_tracer->report_gc_start(heap->gc_cause(), _gc_timer->gc_start());
725
726 _old_gen = heap->old_gen();
727
728 // If the next generation is too full to accommodate promotion
729 // from this generation, pass on collection; let the next generation
730 // do it.
731 if (!collection_attempt_is_safe()) {
732 log_trace(gc)(":: Collection attempt not safe ::");
733 heap->set_incremental_collection_failed(); // Slight lie: we did not even attempt one
734 return;
735 }
736 assert(to()->is_empty(), "Else not collection_attempt_is_safe");
737
738 init_assuming_no_promotion_failure();
739
740 GCTraceTime(Trace, gc, phases) tm("DefNew", nullptr, heap->gc_cause());
741
742 heap->trace_heap_before_gc(_gc_tracer);
743
744 // These can be shared for all code paths
745 IsAliveClosure is_alive(this);
746
747 age_table()->clear();
748 to()->clear(SpaceDecorator::Mangle);
749 // The preserved marks should be empty at the start of the GC.
750 _preserved_marks_set.init(1);
751
752 assert(heap->no_allocs_since_save_marks(),
753 "save marks have not been newly set.");
754
755 YoungGenScanClosure young_gen_cl(this);
756 OldGenScanClosure old_gen_cl(this);
757
758 FastEvacuateFollowersClosure evacuate_followers(heap,
759 &young_gen_cl,
760 &old_gen_cl);
761
762 assert(heap->no_allocs_since_save_marks(),
763 "save marks have not been newly set.");
764
765 {
766 StrongRootsScope srs(0);
767 RootScanClosure root_cl{this};
768 CLDScanClosure cld_scan_closure{this};
769
770 heap->young_process_roots(&root_cl,
771 &old_gen_cl,
772 &cld_scan_closure);
773 }
774
775 // "evacuate followers".
776 evacuate_followers.do_void();
777
778 {
779 // Reference processing
780 KeepAliveClosure keep_alive(this);
781 ReferenceProcessor* rp = ref_processor();
782 ReferenceProcessorPhaseTimes pt(_gc_timer, rp->max_num_queues());
783 SerialGCRefProcProxyTask task(is_alive, keep_alive, evacuate_followers);
784 const ReferenceProcessorStats& stats = rp->process_discovered_references(task, pt);
785 _gc_tracer->report_gc_reference_stats(stats);
786 _gc_tracer->report_tenuring_threshold(tenuring_threshold());
787 pt.print_all_references();
788 }
789 assert(heap->no_allocs_since_save_marks(), "save marks have not been newly set.");
790
791 {
792 AdjustWeakRootClosure cl{this};
793 WeakProcessor::weak_oops_do(&is_alive, &cl);
794 }
795
796 // Verify that the usage of keep_alive didn't copy any objects.
797 assert(heap->no_allocs_since_save_marks(), "save marks have not been newly set.");
798
799 _string_dedup_requests.flush();
800
801 if (!_promotion_failed) {
802 // Swap the survivor spaces.
803 eden()->clear(SpaceDecorator::Mangle);
804 from()->clear(SpaceDecorator::Mangle);
805 if (ZapUnusedHeapArea) {
806 // This is now done here because of the piece-meal mangling which
807 // can check for valid mangling at intermediate points in the
808 // collection(s). When a young collection fails to collect
809 // sufficient space resizing of the young generation can occur
810 // an redistribute the spaces in the young generation. Mangle
811 // here so that unzapped regions don't get distributed to
812 // other spaces.
813 to()->mangle_unused_area();
814 }
815 swap_spaces();
816
817 assert(to()->is_empty(), "to space should be empty now");
818
819 adjust_desired_tenuring_threshold();
820
821 // A successful scavenge should restart the GC time limit count which is
822 // for full GC's.
823 AdaptiveSizePolicy* size_policy = heap->size_policy();
824 size_policy->reset_gc_overhead_limit_count();
825 assert(!heap->incremental_collection_failed(), "Should be clear");
826 } else {
827 assert(_promo_failure_scan_stack.is_empty(), "post condition");
828 _promo_failure_scan_stack.clear(true); // Clear cached segments.
829
830 remove_forwarding_pointers();
831 log_info(gc, promotion)("Promotion failed");
832 // Add to-space to the list of space to compact
833 // when a promotion failure has occurred. In that
834 // case there can be live objects in to-space
835 // as a result of a partial evacuation of eden
836 // and from-space.
837 swap_spaces(); // For uniformity wrt ParNewGeneration.
838 from()->set_next_compaction_space(to());
839 heap->set_incremental_collection_failed();
840
841 // Inform the next generation that a promotion failure occurred.
842 _old_gen->promotion_failure_occurred();
843 _gc_tracer->report_promotion_failed(_promotion_failed_info);
844
845 // Reset the PromotionFailureALot counters.
846 NOT_PRODUCT(heap->reset_promotion_should_fail();)
847 }
848 // We should have processed and cleared all the preserved marks.
849 _preserved_marks_set.reclaim();
850
851 heap->trace_heap_after_gc(_gc_tracer);
852
853 _gc_timer->register_gc_end();
854
855 _gc_tracer->report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions());
856 }
857
858 void DefNewGeneration::init_assuming_no_promotion_failure() {
859 _promotion_failed = false;
860 _promotion_failed_info.reset();
861 from()->set_next_compaction_space(nullptr);
862 }
863
864 void DefNewGeneration::remove_forwarding_pointers() {
865 assert(_promotion_failed, "precondition");
866
867 // Will enter Full GC soon due to failed promotion. Must reset the mark word
868 // of objs in young-gen so that no objs are marked (forwarded) when Full GC
869 // starts. (The mark word is overloaded: `is_marked()` == `is_forwarded()`.)
870 struct ResetForwardedMarkWord : ObjectClosure {
871 void do_object(oop obj) override {
872 if (obj->is_forwarded()) {
873 obj->init_mark();
874 }
875 }
876 } cl;
877 eden()->object_iterate(&cl);
878 from()->object_iterate(&cl);
879
880 restore_preserved_marks();
881 }
882
883 void DefNewGeneration::restore_preserved_marks() {
884 _preserved_marks_set.restore(nullptr);
885 }
886
887 void DefNewGeneration::handle_promotion_failure(oop old) {
888 log_debug(gc, promotion)("Promotion failure size = " SIZE_FORMAT ") ", old->size());
889
890 _promotion_failed = true;
891 _promotion_failed_info.register_copy_failure(old->size());
892 _preserved_marks_set.get()->push_if_necessary(old, old->mark());
893
894 ContinuationGCSupport::transform_stack_chunk(old);
895
896 // forward to self
897 old->forward_to(old);
898
899 _promo_failure_scan_stack.push(old);
900
901 if (!_promo_failure_drain_in_progress) {
902 // prevent recursion in copy_to_survivor_space()
903 _promo_failure_drain_in_progress = true;
904 drain_promo_failure_scan_stack();
905 _promo_failure_drain_in_progress = false;
906 }
907 }
908
909 oop DefNewGeneration::copy_to_survivor_space(oop old) {
910 assert(is_in_reserved(old) && !old->is_forwarded(),
911 "shouldn't be scavenging this oop");
912 size_t s = old->size();
913 oop obj = nullptr;
914
915 // Try allocating obj in to-space (unless too old)
916 if (old->age() < tenuring_threshold()) {
917 obj = cast_to_oop(to()->allocate(s));
918 }
919
920 bool new_obj_is_tenured = false;
921 // Otherwise try allocating obj tenured
922 if (obj == nullptr) {
923 obj = _old_gen->promote(old, s);
924 if (obj == nullptr) {
925 handle_promotion_failure(old);
926 return old;
927 }
928 new_obj_is_tenured = true;
929 } else {
930 // Prefetch beyond obj
931 const intx interval = PrefetchCopyIntervalInBytes;
932 Prefetch::write(obj, interval);
933
934 // Copy obj
935 Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(old), cast_from_oop<HeapWord*>(obj), s);
936
937 ContinuationGCSupport::transform_stack_chunk(obj);
938
939 // Increment age if obj still in new generation
940 obj->incr_age();
941 age_table()->add(obj, s);
942 }
943
944 // Done, insert forward pointer to obj in this header
945 old->forward_to(obj);
946
947 if (SerialStringDedup::is_candidate_from_evacuation(obj, new_obj_is_tenured)) {
948 // Record old; request adds a new weak reference, which reference
949 // processing expects to refer to a from-space object.
950 _string_dedup_requests.add(old);
951 }
952 return obj;
953 }
954
955 void DefNewGeneration::drain_promo_failure_scan_stack() {
956 PromoteFailureClosure cl{this};
957 while (!_promo_failure_scan_stack.is_empty()) {
958 oop obj = _promo_failure_scan_stack.pop();
959 obj->oop_iterate(&cl);
960 }
961 }
962
963 void DefNewGeneration::save_marks() {
964 eden()->set_saved_mark();
965 to()->set_saved_mark();
966 from()->set_saved_mark();
967 }
968
969
970 bool DefNewGeneration::no_allocs_since_save_marks() {
971 assert(eden()->saved_mark_at_top(), "Violated spec - alloc in eden");
972 assert(from()->saved_mark_at_top(), "Violated spec - alloc in from");
973 return to()->saved_mark_at_top();
974 }
975
976 void DefNewGeneration::contribute_scratch(ScratchBlock*& list, Generation* requestor,
977 size_t max_alloc_words) {
978 if (requestor == this || _promotion_failed) {
979 return;
980 }
981 assert(GenCollectedHeap::heap()->is_old_gen(requestor), "We should not call our own generation");
982
983 /* $$$ Assert this? "trace" is a "MarkSweep" function so that's not appropriate.
984 if (to_space->top() > to_space->bottom()) {
985 trace("to_space not empty when contribute_scratch called");
986 }
987 */
988
989 ContiguousSpace* to_space = to();
990 assert(to_space->end() >= to_space->top(), "pointers out of order");
991 size_t free_words = pointer_delta(to_space->end(), to_space->top());
992 if (free_words >= MinFreeScratchWords) {
993 ScratchBlock* sb = (ScratchBlock*)to_space->top();
994 sb->num_words = free_words;
995 sb->next = list;
996 list = sb;
997 }
998 }
999
1000 void DefNewGeneration::reset_scratch() {
1001 // If contributing scratch in to_space, mangle all of
1002 // to_space if ZapUnusedHeapArea. This is needed because
1003 // top is not maintained while using to-space as scratch.
1004 if (ZapUnusedHeapArea) {
1005 to()->mangle_unused_area_complete();
1006 }
1007 }
1008
1009 bool DefNewGeneration::collection_attempt_is_safe() {
1010 if (!to()->is_empty()) {
1011 log_trace(gc)(":: to is not empty ::");
1012 return false;
1013 }
1014 if (_old_gen == nullptr) {
1015 GenCollectedHeap* gch = GenCollectedHeap::heap();
1016 _old_gen = gch->old_gen();
1017 }
1018 return _old_gen->promotion_attempt_is_safe(used());
1019 }
1020
1021 void DefNewGeneration::gc_epilogue(bool full) {
1022 DEBUG_ONLY(static bool seen_incremental_collection_failed = false;)
1023
1024 assert(!GCLocker::is_active(), "We should not be executing here");
1025 // Check if the heap is approaching full after a collection has
1026 // been done. Generally the young generation is empty at
1027 // a minimum at the end of a collection. If it is not, then
1028 // the heap is approaching full.
1029 GenCollectedHeap* gch = GenCollectedHeap::heap();
1030 if (full) {
1031 DEBUG_ONLY(seen_incremental_collection_failed = false;)
1032 if (!collection_attempt_is_safe() && !_eden_space->is_empty()) {
1033 log_trace(gc)("DefNewEpilogue: cause(%s), full, not safe, set_failed, set_alloc_from, clear_seen",
1034 GCCause::to_string(gch->gc_cause()));
1035 gch->set_incremental_collection_failed(); // Slight lie: a full gc left us in that state
1036 set_should_allocate_from_space(); // we seem to be running out of space
1037 } else {
1038 log_trace(gc)("DefNewEpilogue: cause(%s), full, safe, clear_failed, clear_alloc_from, clear_seen",
1039 GCCause::to_string(gch->gc_cause()));
1040 gch->clear_incremental_collection_failed(); // We just did a full collection
1041 clear_should_allocate_from_space(); // if set
1042 }
1043 } else {
1044 #ifdef ASSERT
1045 // It is possible that incremental_collection_failed() == true
1046 // here, because an attempted scavenge did not succeed. The policy
1047 // is normally expected to cause a full collection which should
1048 // clear that condition, so we should not be here twice in a row
1049 // with incremental_collection_failed() == true without having done
1050 // a full collection in between.
1051 if (!seen_incremental_collection_failed &&
1052 gch->incremental_collection_failed()) {
1053 log_trace(gc)("DefNewEpilogue: cause(%s), not full, not_seen_failed, failed, set_seen_failed",
1054 GCCause::to_string(gch->gc_cause()));
1055 seen_incremental_collection_failed = true;
1056 } else if (seen_incremental_collection_failed) {
1057 log_trace(gc)("DefNewEpilogue: cause(%s), not full, seen_failed, will_clear_seen_failed",
1058 GCCause::to_string(gch->gc_cause()));
1059 seen_incremental_collection_failed = false;
1060 }
1061 #endif // ASSERT
1062 }
1063
1064 if (ZapUnusedHeapArea) {
1065 eden()->check_mangled_unused_area_complete();
1066 from()->check_mangled_unused_area_complete();
1067 to()->check_mangled_unused_area_complete();
1068 }
1069
1070 // update the generation and space performance counters
1071 update_counters();
1072 gch->counters()->update_counters();
1073 }
1074
1075 void DefNewGeneration::record_spaces_top() {
1076 assert(ZapUnusedHeapArea, "Not mangling unused space");
1077 eden()->set_top_for_allocations();
1078 to()->set_top_for_allocations();
1079 from()->set_top_for_allocations();
1080 }
1081
1082 void DefNewGeneration::update_counters() {
1083 if (UsePerfData) {
1084 _eden_counters->update_all();
1085 _from_counters->update_all();
1086 _to_counters->update_all();
1087 _gen_counters->update_all();
1088 }
1089 }
1090
1091 void DefNewGeneration::verify() {
1092 eden()->verify();
1093 from()->verify();
1094 to()->verify();
1095 }
1096
1097 void DefNewGeneration::print_on(outputStream* st) const {
1098 Generation::print_on(st);
1099 st->print(" eden");
1100 eden()->print_on(st);
1101 st->print(" from");
1102 from()->print_on(st);
1103 st->print(" to ");
1104 to()->print_on(st);
1105 }
1106
1107
1108 const char* DefNewGeneration::name() const {
1109 return "def new generation";
1110 }
1111
1112 // Moved from inline file as they are not called inline
1113 CompactibleSpace* DefNewGeneration::first_compaction_space() const {
1114 return eden();
1115 }
1116
1117 HeapWord* DefNewGeneration::allocate(size_t word_size, bool is_tlab) {
1118 // This is the slow-path allocation for the DefNewGeneration.
1119 // Most allocations are fast-path in compiled code.
1120 // We try to allocate from the eden. If that works, we are happy.
1121 // Note that since DefNewGeneration supports lock-free allocation, we
1122 // have to use it here, as well.
1123 HeapWord* result = eden()->par_allocate(word_size);
1124 if (result == nullptr) {
1125 // If the eden is full and the last collection bailed out, we are running
1126 // out of heap space, and we try to allocate the from-space, too.
1127 // allocate_from_space can't be inlined because that would introduce a
1128 // circular dependency at compile time.
1129 result = allocate_from_space(word_size);
1130 }
1131 return result;
1132 }
1133
1134 HeapWord* DefNewGeneration::par_allocate(size_t word_size,
1135 bool is_tlab) {
1136 return eden()->par_allocate(word_size);
1137 }
1138
1139 size_t DefNewGeneration::tlab_capacity() const {
1140 return eden()->capacity();
1141 }
1142
1143 size_t DefNewGeneration::tlab_used() const {
1144 return eden()->used();
1145 }
1146
1147 size_t DefNewGeneration::unsafe_max_tlab_alloc() const {
1148 return unsafe_max_alloc_nogc();
1149 }