1 /*
2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
28 #include "gc/shenandoah/shenandoahAgeCensus.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30
31 ShenandoahAgeCensus::ShenandoahAgeCensus() {
32 assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only in generational mode");
33 if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) {
34 vm_exit_during_initialization(
35 err_msg("ShenandoahGenerationalMinTenuringAge=" SIZE_FORMAT
36 " should be no more than ShenandoahGenerationalMaxTenuringAge=" SIZE_FORMAT,
37 ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge));
38 }
39
40 _global_age_table = NEW_C_HEAP_ARRAY(AgeTable*, MAX_SNAPSHOTS, mtGC);
41 CENSUS_NOISE(_global_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, MAX_SNAPSHOTS, mtGC);)
42 _tenuring_threshold = NEW_C_HEAP_ARRAY(uint, MAX_SNAPSHOTS, mtGC);
43
44 for (int i = 0; i < MAX_SNAPSHOTS; i++) {
45 // Note that we don't now get perfdata from age_table
46 _global_age_table[i] = new AgeTable(false);
47 CENSUS_NOISE(_global_noise[i].clear();)
48 // Sentinel value
49 _tenuring_threshold[i] = MAX_COHORTS;
50 }
51 if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
52 size_t max_workers = ShenandoahHeap::heap()->max_workers();
53 _local_age_table = NEW_C_HEAP_ARRAY(AgeTable*, max_workers, mtGC);
54 CENSUS_NOISE(_local_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, max_workers, mtGC);)
55 for (uint i = 0; i < max_workers; i++) {
56 _local_age_table[i] = new AgeTable(false);
57 CENSUS_NOISE(_local_noise[i].clear();)
58 }
59 } else {
60 _local_age_table = nullptr;
61 }
62 _epoch = MAX_SNAPSHOTS - 1; // see update_epoch()
63 }
64
65 CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {)
66 NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {)
67 if (obj_age <= markWord::max_age) {
68 assert(obj_age < MAX_COHORTS && region_age < MAX_COHORTS, "Should have been tenured");
69 #ifdef SHENANDOAH_CENSUS_NOISE
70 // Region ageing is stochastic and non-monotonic; this vitiates mortality
71 // demographics in ways that might defeat our algorithms. Marking may be a
72 // time when we might be able to correct this, but we currently do not do
73 // this. Like skipped statistics further below, we want to track the
74 // impact of this noise to see if this may be worthwhile. JDK-<TBD>.
75 uint age = obj_age;
76 if (region_age > 0) {
77 add_aged(size, worker_id); // this tracking is coarse for now
78 age += region_age;
79 if (age >= MAX_COHORTS) {
80 age = (uint)(MAX_COHORTS - 1); // clamp
81 add_clamped(size, worker_id);
82 }
83 }
84 if (region_youth > 0) { // track object volume with retrograde age
85 add_young(size, worker_id);
86 }
87 #else // SHENANDOAH_CENSUS_NOISE
88 uint age = MIN2(obj_age + region_age, (uint)(MAX_COHORTS - 1)); // clamp
89 #endif // SHENANDOAH_CENSUS_NOISE
90 get_local_age_table(worker_id)->add(age, size);
91 } else {
92 // update skipped statistics
93 CENSUS_NOISE(add_skipped(size, worker_id);)
94 }
95 }
96
97 #ifdef SHENANDOAH_CENSUS_NOISE
98 void ShenandoahAgeCensus::add_skipped(size_t size, uint worker_id) {
99 _local_noise[worker_id].skipped += size;
100 }
101
102 void ShenandoahAgeCensus::add_aged(size_t size, uint worker_id) {
103 _local_noise[worker_id].aged += size;
104 }
105
106 void ShenandoahAgeCensus::add_clamped(size_t size, uint worker_id) {
107 _local_noise[worker_id].clamped += size;
108 }
109
110 void ShenandoahAgeCensus::add_young(size_t size, uint worker_id) {
111 _local_noise[worker_id].young += size;
112 }
113 #endif // SHENANDOAH_CENSUS_NOISE
114
115 // Prepare for a new census update, by clearing appropriate global slots.
116 void ShenandoahAgeCensus::prepare_for_census_update() {
117 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
118 if (++_epoch >= MAX_SNAPSHOTS) {
119 _epoch=0;
120 }
121 _global_age_table[_epoch]->clear();
122 CENSUS_NOISE(_global_noise[_epoch].clear();)
123 }
124
125 // Update the census data from appropriate sources,
126 // and compute the new tenuring threshold.
127 void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable* pv2) {
128 prepare_for_census_update();
129 assert(_global_age_table[_epoch]->is_clear(), "Dirty decks");
130 CENSUS_NOISE(assert(_global_noise[_epoch].is_clear(), "Dirty decks");)
131 if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
132 assert(pv1 == nullptr && pv2 == nullptr, "Error, check caller");
133 // Seed cohort 0 with population that may have been missed during
134 // regular census.
135 _global_age_table[_epoch]->add((uint)0, age0_pop);
136
137 size_t max_workers = ShenandoahHeap::heap()->max_workers();
138 // Merge data from local age tables into the global age table for the epoch,
139 // clearing the local tables.
140 for (uint i = 0; i < max_workers; i++) {
141 // age stats
142 _global_age_table[_epoch]->merge(_local_age_table[i]);
143 _local_age_table[i]->clear(); // clear for next census
144 // Merge noise stats
145 CENSUS_NOISE(_global_noise[_epoch].merge(_local_noise[i]);)
146 CENSUS_NOISE(_local_noise[i].clear();)
147 }
148 } else {
149 // census during evac
150 assert(pv1 != nullptr && pv2 != nullptr, "Error, check caller");
151 _global_age_table[_epoch]->merge(pv1);
152 _global_age_table[_epoch]->merge(pv2);
153 }
154
155 update_tenuring_threshold();
156
157 // used for checking reasonableness of census coverage, non-product
158 // only.
159 NOT_PRODUCT(update_total();)
160 }
161
162
163 // Reset the epoch for the global age tables,
164 // clearing all history.
165 void ShenandoahAgeCensus::reset_global() {
166 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
167 for (uint i = 0; i < MAX_SNAPSHOTS; i++) {
168 _global_age_table[i]->clear();
169 CENSUS_NOISE(_global_noise[i].clear();)
170 }
171 _epoch = MAX_SNAPSHOTS;
172 assert(_epoch < MAX_SNAPSHOTS, "Error");
173 }
174
175 // Reset the local age tables, clearing any partial census.
176 void ShenandoahAgeCensus::reset_local() {
177 if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) {
178 assert(_local_age_table == nullptr, "Error");
179 return;
180 }
181 size_t max_workers = ShenandoahHeap::heap()->max_workers();
182 for (uint i = 0; i < max_workers; i++) {
183 _local_age_table[i]->clear();
184 CENSUS_NOISE(_local_noise[i].clear();)
185 }
186 }
187
188 #ifndef PRODUCT
189 // Is global census information clear?
190 bool ShenandoahAgeCensus::is_clear_global() {
191 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
192 for (uint i = 0; i < MAX_SNAPSHOTS; i++) {
193 bool clear = _global_age_table[i]->is_clear();
194 CENSUS_NOISE(clear |= _global_noise[i].is_clear();)
195 if (!clear) {
196 return false;
197 }
198 }
199 return true;
200 }
201
202 // Is local census information clear?
203 bool ShenandoahAgeCensus::is_clear_local() {
204 if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) {
205 assert(_local_age_table == nullptr, "Error");
206 return true;
207 }
208 size_t max_workers = ShenandoahHeap::heap()->max_workers();
209 for (uint i = 0; i < max_workers; i++) {
210 bool clear = _local_age_table[i]->is_clear();
211 CENSUS_NOISE(clear |= _local_noise[i].is_clear();)
212 if (!clear) {
213 return false;
214 }
215 }
216 return true;
217 }
218
219 size_t ShenandoahAgeCensus::get_all_ages(uint snap) {
220 assert(snap < MAX_SNAPSHOTS, "Out of bounds");
221 size_t pop = 0;
222 const AgeTable* pv = _global_age_table[snap];
223 for (uint i = 0; i < MAX_COHORTS; i++) {
224 pop += pv->sizes[i];
225 }
226 return pop;
227 }
228
229 size_t ShenandoahAgeCensus::get_skipped(uint snap) {
230 assert(snap < MAX_SNAPSHOTS, "Out of bounds");
231 return _global_noise[snap].skipped;
232 }
233
234 void ShenandoahAgeCensus::update_total() {
235 _counted = get_all_ages(_epoch);
236 _skipped = get_skipped(_epoch);
237 _total = _counted + _skipped;
238 }
239 #endif // !PRODUCT
240
241 void ShenandoahAgeCensus::update_tenuring_threshold() {
242 if (!ShenandoahGenerationalAdaptiveTenuring) {
243 _tenuring_threshold[_epoch] = InitialTenuringThreshold;
244 } else {
245 uint tt = compute_tenuring_threshold();
246 assert(tt <= MAX_COHORTS, "Out of bounds");
247 _tenuring_threshold[_epoch] = tt;
248 }
249 print();
250 log_trace(gc, age)("New tenuring threshold " UINTX_FORMAT " (min " UINTX_FORMAT ", max " UINTX_FORMAT")",
251 (uintx) _tenuring_threshold[_epoch], ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge);
252 }
253
254 // Currently Shenandoah{Min,Max}TenuringAge have a floor of 1 because we
255 // aren't set up to promote age 0 objects.
256 uint ShenandoahAgeCensus::compute_tenuring_threshold() {
257 // Dispose of the extremal cases early so the loop below
258 // is less fragile.
259 if (ShenandoahGenerationalMaxTenuringAge == ShenandoahGenerationalMinTenuringAge) {
260 return ShenandoahGenerationalMaxTenuringAge; // Any value in [1,16]
261 }
262 assert(ShenandoahGenerationalMinTenuringAge < ShenandoahGenerationalMaxTenuringAge, "Error");
263
264 // Starting with the oldest cohort with a non-trivial population
265 // (as specified by ShenandoahGenerationalTenuringCohortPopulationThreshold) in the
266 // previous epoch, and working down the cohorts by age, find the
267 // oldest age that has a significant mortality rate (as specified by
268 // ShenandoahGenerationalTenuringMortalityRateThreshold). We use this as
269 // tenuring age to be used for the evacuation cycle to follow.
270 // Results are clamped between user-specified min & max guardrails,
271 // so we ignore any cohorts outside ShenandoahGenerational[Min,Max]Age.
272
273 // Current and previous epoch in ring
274 const uint cur_epoch = _epoch;
275 const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1 : markWord::max_age;
276
277 // Current and previous population vectors in ring
278 const AgeTable* cur_pv = _global_age_table[cur_epoch];
279 const AgeTable* prev_pv = _global_age_table[prev_epoch];
280 uint upper_bound = ShenandoahGenerationalMaxTenuringAge;
281 const uint prev_tt = previous_tenuring_threshold();
282 if (ShenandoahGenerationalCensusIgnoreOlderCohorts && prev_tt > 0) {
283 // We stay below the computed tenuring threshold for the last cycle plus 1,
284 // ignoring the mortality rates of any older cohorts.
285 upper_bound = MIN2(upper_bound, prev_tt + 1);
286 }
287 upper_bound = MIN2(upper_bound, markWord::max_age);
288
289 const uint lower_bound = MAX2((uint)ShenandoahGenerationalMinTenuringAge, (uint)1);
290
291 uint tenuring_threshold = upper_bound;
292 for (uint i = upper_bound; i >= lower_bound; i--) {
293 assert(i > 0, "Index (i-1) would underflow/wrap");
294 assert(i <= markWord::max_age, "Index i would overflow");
295 // Cohort of current age i
296 const size_t cur_pop = cur_pv->sizes[i];
297 const size_t prev_pop = prev_pv->sizes[i-1];
298 const double mr = mortality_rate(prev_pop, cur_pop);
299 if (prev_pop > ShenandoahGenerationalTenuringCohortPopulationThreshold &&
300 mr > ShenandoahGenerationalTenuringMortalityRateThreshold) {
301 // This is the oldest cohort that has high mortality.
302 // We ignore any cohorts that had a very low population count, or
303 // that have a lower mortality rate than we care to age in young; these
304 // cohorts are considered eligible for tenuring when all older
305 // cohorts are. We return the next higher age as the tenuring threshold
306 // so that we do not prematurely promote objects of this age.
307 assert(tenuring_threshold == i+1 || tenuring_threshold == upper_bound, "Error");
308 assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error");
309 return tenuring_threshold;
310 }
311 // Remember that we passed over this cohort, looking for younger cohorts
312 // showing high mortality. We want to tenure cohorts of this age.
313 tenuring_threshold = i;
314 }
315 assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error");
316 return tenuring_threshold;
317 }
318
319 // Mortality rate of a cohort, given its previous and current population
320 double ShenandoahAgeCensus::mortality_rate(size_t prev_pop, size_t cur_pop) {
321 // The following also covers the case where both entries are 0
322 if (prev_pop <= cur_pop) {
323 // adjust for inaccurate censuses by finessing the
324 // reappearance of dark matter as normal matter;
325 // mortality rate is 0 if population remained the same
326 // or increased.
327 if (cur_pop > prev_pop) {
328 log_trace(gc, age)
329 (" (dark matter) Cohort population " SIZE_FORMAT_W(10) " to " SIZE_FORMAT_W(10),
330 prev_pop*oopSize, cur_pop*oopSize);
331 }
332 return 0.0;
333 }
334 assert(prev_pop > 0 && prev_pop > cur_pop, "Error");
335 return 1.0 - (((double)cur_pop)/((double)prev_pop));
336 }
337
338 void ShenandoahAgeCensus::print() {
339 // Print the population vector for the current epoch, and
340 // for the previous epoch, as well as the computed mortality
341 // ratio for each extant cohort.
342 const uint cur_epoch = _epoch;
343 const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1: markWord::max_age;
344
345 const AgeTable* cur_pv = _global_age_table[cur_epoch];
346 const AgeTable* prev_pv = _global_age_table[prev_epoch];
347
348 const uint tt = tenuring_threshold();
349
350 size_t total= 0;
351 for (uint i = 1; i < MAX_COHORTS; i++) {
352 const size_t prev_pop = prev_pv->sizes[i-1]; // (i-1) OK because i >= 1
353 const size_t cur_pop = cur_pv->sizes[i];
354 double mr = mortality_rate(prev_pop, cur_pop);
355 // Suppress printing when everything is zero
356 if (prev_pop + cur_pop > 0) {
357 log_info(gc, age)
358 (" - age %3u: prev " SIZE_FORMAT_W(10) " bytes, curr " SIZE_FORMAT_W(10) " bytes, mortality %.2f ",
359 i, prev_pop*oopSize, cur_pop*oopSize, mr);
360 }
361 total += cur_pop;
362 if (i == tt) {
363 // Underline the cohort for tenuring threshold (if < MAX_COHORTS)
364 log_info(gc, age)("----------------------------------------------------------------------------");
365 }
366 }
367 CENSUS_NOISE(_global_noise[cur_epoch].print(total);)
368 }
369
370 #ifdef SHENANDOAH_CENSUS_NOISE
371 void ShenandoahNoiseStats::print(size_t total) {
372 if (total > 0) {
373 float f_skipped = (float)skipped/(float)total;
374 float f_aged = (float)aged/(float)total;
375 float f_clamped = (float)clamped/(float)total;
376 float f_young = (float)young/(float)total;
377 log_info(gc, age)("Skipped: " SIZE_FORMAT_W(10) " (%.2f), R-Aged: " SIZE_FORMAT_W(10) " (%.2f), "
378 "Clamped: " SIZE_FORMAT_W(10) " (%.2f), R-Young: " SIZE_FORMAT_W(10) " (%.2f)",
379 skipped*oopSize, f_skipped, aged*oopSize, f_aged,
380 clamped*oopSize, f_clamped, young*oopSize, f_young);
381 }
382 }
383 #endif // SHENANDOAH_CENSUS_NOISE