1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" 28 #include "gc/shenandoah/shenandoahAgeCensus.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 31 ShenandoahAgeCensus::ShenandoahAgeCensus() 32 : ShenandoahAgeCensus(ShenandoahHeap::heap()->max_workers()) 33 { 34 assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only in generational mode"); 35 } 36 37 ShenandoahAgeCensus::ShenandoahAgeCensus(uint max_workers) 38 : _max_workers(max_workers) 39 { 40 if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) { 41 vm_exit_during_initialization( 42 err_msg("ShenandoahGenerationalMinTenuringAge=" SIZE_FORMAT 43 " should be no more than ShenandoahGenerationalMaxTenuringAge=" SIZE_FORMAT, 44 ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge)); 45 } 46 47 _global_age_table = NEW_C_HEAP_ARRAY(AgeTable*, MAX_SNAPSHOTS, mtGC); 48 CENSUS_NOISE(_global_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, MAX_SNAPSHOTS, mtGC);) 49 _tenuring_threshold = NEW_C_HEAP_ARRAY(uint, MAX_SNAPSHOTS, mtGC); 50 CENSUS_NOISE(_skipped = 0); 51 NOT_PRODUCT(_counted = 0); 52 NOT_PRODUCT(_total = 0); 53 54 for (int i = 0; i < MAX_SNAPSHOTS; i++) { 55 // Note that we don't now get perfdata from age_table 56 _global_age_table[i] = new AgeTable(false); 57 CENSUS_NOISE(_global_noise[i].clear();) 58 // Sentinel value 59 _tenuring_threshold[i] = MAX_COHORTS; 60 } 61 if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) { 62 _local_age_table = NEW_C_HEAP_ARRAY(AgeTable*, _max_workers, mtGC); 63 CENSUS_NOISE(_local_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, max_workers, mtGC);) 64 for (uint i = 0; i < _max_workers; i++) { 65 _local_age_table[i] = new AgeTable(false); 66 CENSUS_NOISE(_local_noise[i].clear();) 67 } 68 } else { 69 _local_age_table = nullptr; 70 } 71 _epoch = MAX_SNAPSHOTS - 1; // see update_epoch() 72 } 73 74 ShenandoahAgeCensus::~ShenandoahAgeCensus() { 75 for (uint i = 0; i < MAX_SNAPSHOTS; i++) { 76 delete _global_age_table[i]; 77 } 78 FREE_C_HEAP_ARRAY(AgeTable*, _global_age_table); 79 FREE_C_HEAP_ARRAY(uint, _tenuring_threshold); 80 CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _global_noise)); 81 if (_local_age_table) { 82 for (uint i = 0; i < _max_workers; i++) { 83 delete _local_age_table[i]; 84 } 85 FREE_C_HEAP_ARRAY(AgeTable*, _local_age_table); 86 CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _local_noise)); 87 } 88 } 89 90 CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {) 91 NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {) 92 if (obj_age <= markWord::max_age) { 93 assert(obj_age < MAX_COHORTS && region_age < MAX_COHORTS, "Should have been tenured"); 94 #ifdef SHENANDOAH_CENSUS_NOISE 95 // Region ageing is stochastic and non-monotonic; this vitiates mortality 96 // demographics in ways that might defeat our algorithms. Marking may be a 97 // time when we might be able to correct this, but we currently do not do 98 // this. Like skipped statistics further below, we want to track the 99 // impact of this noise to see if this may be worthwhile. JDK-<TBD>. 100 uint age = obj_age; 101 if (region_age > 0) { 102 add_aged(size, worker_id); // this tracking is coarse for now 103 age += region_age; 104 if (age >= MAX_COHORTS) { 105 age = (uint)(MAX_COHORTS - 1); // clamp 106 add_clamped(size, worker_id); 107 } 108 } 109 if (region_youth > 0) { // track object volume with retrograde age 110 add_young(size, worker_id); 111 } 112 #else // SHENANDOAH_CENSUS_NOISE 113 uint age = MIN2(obj_age + region_age, (uint)(MAX_COHORTS - 1)); // clamp 114 #endif // SHENANDOAH_CENSUS_NOISE 115 get_local_age_table(worker_id)->add(age, size); 116 } else { 117 // update skipped statistics 118 CENSUS_NOISE(add_skipped(size, worker_id);) 119 } 120 } 121 122 #ifdef SHENANDOAH_CENSUS_NOISE 123 void ShenandoahAgeCensus::add_skipped(size_t size, uint worker_id) { 124 _local_noise[worker_id].skipped += size; 125 } 126 127 void ShenandoahAgeCensus::add_aged(size_t size, uint worker_id) { 128 _local_noise[worker_id].aged += size; 129 } 130 131 void ShenandoahAgeCensus::add_clamped(size_t size, uint worker_id) { 132 _local_noise[worker_id].clamped += size; 133 } 134 135 void ShenandoahAgeCensus::add_young(size_t size, uint worker_id) { 136 _local_noise[worker_id].young += size; 137 } 138 #endif // SHENANDOAH_CENSUS_NOISE 139 140 // Prepare for a new census update, by clearing appropriate global slots. 141 void ShenandoahAgeCensus::prepare_for_census_update() { 142 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds"); 143 if (++_epoch >= MAX_SNAPSHOTS) { 144 _epoch=0; 145 } 146 _global_age_table[_epoch]->clear(); 147 CENSUS_NOISE(_global_noise[_epoch].clear();) 148 } 149 150 // Update the census data from appropriate sources, 151 // and compute the new tenuring threshold. 152 void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable* pv2) { 153 prepare_for_census_update(); 154 assert(_global_age_table[_epoch]->is_clear(), "Dirty decks"); 155 CENSUS_NOISE(assert(_global_noise[_epoch].is_clear(), "Dirty decks");) 156 if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) { 157 assert(pv1 == nullptr && pv2 == nullptr, "Error, check caller"); 158 // Seed cohort 0 with population that may have been missed during 159 // regular census. 160 _global_age_table[_epoch]->add(0u, age0_pop); 161 162 // Merge data from local age tables into the global age table for the epoch, 163 // clearing the local tables. 164 for (uint i = 0; i < _max_workers; i++) { 165 // age stats 166 _global_age_table[_epoch]->merge(_local_age_table[i]); 167 _local_age_table[i]->clear(); // clear for next census 168 // Merge noise stats 169 CENSUS_NOISE(_global_noise[_epoch].merge(_local_noise[i]);) 170 CENSUS_NOISE(_local_noise[i].clear();) 171 } 172 } else { 173 // census during evac 174 assert(pv1 != nullptr && pv2 != nullptr, "Error, check caller"); 175 _global_age_table[_epoch]->merge(pv1); 176 _global_age_table[_epoch]->merge(pv2); 177 } 178 179 update_tenuring_threshold(); 180 181 // used for checking reasonableness of census coverage, non-product 182 // only. 183 NOT_PRODUCT(update_total();) 184 } 185 186 187 // Reset the epoch for the global age tables, 188 // clearing all history. 189 void ShenandoahAgeCensus::reset_global() { 190 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds"); 191 for (uint i = 0; i < MAX_SNAPSHOTS; i++) { 192 _global_age_table[i]->clear(); 193 CENSUS_NOISE(_global_noise[i].clear();) 194 } 195 _epoch = MAX_SNAPSHOTS; 196 assert(_epoch < MAX_SNAPSHOTS, "Error"); 197 } 198 199 // Reset the local age tables, clearing any partial census. 200 void ShenandoahAgeCensus::reset_local() { 201 if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) { 202 assert(_local_age_table == nullptr, "Error"); 203 return; 204 } 205 for (uint i = 0; i < _max_workers; i++) { 206 _local_age_table[i]->clear(); 207 CENSUS_NOISE(_local_noise[i].clear();) 208 } 209 } 210 211 #ifndef PRODUCT 212 // Is global census information clear? 213 bool ShenandoahAgeCensus::is_clear_global() { 214 assert(_epoch < MAX_SNAPSHOTS, "Out of bounds"); 215 for (uint i = 0; i < MAX_SNAPSHOTS; i++) { 216 bool clear = _global_age_table[i]->is_clear(); 217 CENSUS_NOISE(clear |= _global_noise[i].is_clear();) 218 if (!clear) { 219 return false; 220 } 221 } 222 return true; 223 } 224 225 // Is local census information clear? 226 bool ShenandoahAgeCensus::is_clear_local() { 227 if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) { 228 assert(_local_age_table == nullptr, "Error"); 229 return true; 230 } 231 for (uint i = 0; i < _max_workers; i++) { 232 bool clear = _local_age_table[i]->is_clear(); 233 CENSUS_NOISE(clear |= _local_noise[i].is_clear();) 234 if (!clear) { 235 return false; 236 } 237 } 238 return true; 239 } 240 241 size_t ShenandoahAgeCensus::get_all_ages(uint snap) { 242 assert(snap < MAX_SNAPSHOTS, "Out of bounds"); 243 size_t pop = 0; 244 const AgeTable* pv = _global_age_table[snap]; 245 for (uint i = 0; i < MAX_COHORTS; i++) { 246 pop += pv->sizes[i]; 247 } 248 return pop; 249 } 250 251 size_t ShenandoahAgeCensus::get_skipped(uint snap) { 252 assert(snap < MAX_SNAPSHOTS, "Out of bounds"); 253 return _global_noise[snap].skipped; 254 } 255 256 void ShenandoahAgeCensus::update_total() { 257 _counted = get_all_ages(_epoch); 258 _skipped = get_skipped(_epoch); 259 _total = _counted + _skipped; 260 } 261 #endif // !PRODUCT 262 263 void ShenandoahAgeCensus::update_tenuring_threshold() { 264 if (!ShenandoahGenerationalAdaptiveTenuring) { 265 _tenuring_threshold[_epoch] = InitialTenuringThreshold; 266 } else { 267 uint tt = compute_tenuring_threshold(); 268 assert(tt <= MAX_COHORTS, "Out of bounds"); 269 _tenuring_threshold[_epoch] = tt; 270 } 271 print(); 272 log_info(gc, age)("New tenuring threshold " UINTX_FORMAT " (min " UINTX_FORMAT ", max " UINTX_FORMAT")", 273 (uintx) _tenuring_threshold[_epoch], ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge); 274 } 275 276 // Currently Shenandoah{Min,Max}TenuringAge have a floor of 1 because we 277 // aren't set up to promote age 0 objects. 278 uint ShenandoahAgeCensus::compute_tenuring_threshold() { 279 // Dispose of the extremal cases early so the loop below 280 // is less fragile. 281 if (ShenandoahGenerationalMaxTenuringAge == ShenandoahGenerationalMinTenuringAge) { 282 return ShenandoahGenerationalMaxTenuringAge; // Any value in [1,16] 283 } 284 assert(ShenandoahGenerationalMinTenuringAge < ShenandoahGenerationalMaxTenuringAge, "Error"); 285 286 // Starting with the oldest cohort with a non-trivial population 287 // (as specified by ShenandoahGenerationalTenuringCohortPopulationThreshold) in the 288 // previous epoch, and working down the cohorts by age, find the 289 // oldest age that has a significant mortality rate (as specified by 290 // ShenandoahGenerationalTenuringMortalityRateThreshold). We use this as 291 // tenuring age to be used for the evacuation cycle to follow. 292 // Results are clamped between user-specified min & max guardrails, 293 // so we ignore any cohorts outside ShenandoahGenerational[Min,Max]Age. 294 295 // Current and previous epoch in ring 296 const uint cur_epoch = _epoch; 297 const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1 : markWord::max_age; 298 299 // Current and previous population vectors in ring 300 const AgeTable* cur_pv = _global_age_table[cur_epoch]; 301 const AgeTable* prev_pv = _global_age_table[prev_epoch]; 302 uint upper_bound = ShenandoahGenerationalMaxTenuringAge; 303 const uint prev_tt = previous_tenuring_threshold(); 304 if (ShenandoahGenerationalCensusIgnoreOlderCohorts && prev_tt > 0) { 305 // We stay below the computed tenuring threshold for the last cycle, 306 // ignoring the mortality rates of any older cohorts (which may see 307 // higher mortality rates due to promotions). 308 upper_bound = MIN2(upper_bound, prev_tt); 309 } 310 upper_bound = MIN2(upper_bound, markWord::max_age); 311 312 const uint lower_bound = MAX2((uint)ShenandoahGenerationalMinTenuringAge, 1u); 313 314 uint tenuring_threshold = upper_bound; 315 for (uint i = upper_bound; i >= lower_bound; i--) { 316 assert(i > 0, "Index (i-1) would underflow/wrap"); 317 assert(i <= markWord::max_age, "Index i would overflow"); 318 // Cohort of current age i 319 const size_t cur_pop = cur_pv->sizes[i]; 320 const size_t prev_pop = prev_pv->sizes[i-1]; 321 const double mr = mortality_rate(prev_pop, cur_pop); 322 if (prev_pop > ShenandoahGenerationalTenuringCohortPopulationThreshold && 323 mr > ShenandoahGenerationalTenuringMortalityRateThreshold) { 324 // This is the oldest cohort that has high mortality. 325 // We ignore any cohorts that had a very low population count, or 326 // that have a lower mortality rate than we care to age in young; these 327 // cohorts are considered eligible for tenuring when all older 328 // cohorts are. We return the next higher age as the tenuring threshold 329 // so that we do not prematurely promote objects of this age. 330 assert(tenuring_threshold == i + 1 || tenuring_threshold == upper_bound, "Error"); 331 assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error"); 332 return i + 1; 333 } 334 // Remember that we passed over this cohort, looking for younger cohorts 335 // showing high mortality. We want to tenure cohorts of this age. 336 tenuring_threshold = i; 337 } 338 assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error"); 339 return tenuring_threshold; 340 } 341 342 // Mortality rate of a cohort, given its previous and current population 343 double ShenandoahAgeCensus::mortality_rate(size_t prev_pop, size_t cur_pop) { 344 // The following also covers the case where both entries are 0 345 if (prev_pop <= cur_pop) { 346 // adjust for inaccurate censuses by finessing the 347 // reappearance of dark matter as normal matter; 348 // mortality rate is 0 if population remained the same 349 // or increased. 350 if (cur_pop > prev_pop) { 351 log_trace(gc, age) 352 (" (dark matter) Cohort population " SIZE_FORMAT_W(10) " to " SIZE_FORMAT_W(10), 353 prev_pop*oopSize, cur_pop*oopSize); 354 } 355 return 0.0; 356 } 357 assert(prev_pop > 0 && prev_pop > cur_pop, "Error"); 358 return 1.0 - (((double)cur_pop)/((double)prev_pop)); 359 } 360 361 void ShenandoahAgeCensus::print() { 362 363 const LogTarget(Debug, gc, age) lt; 364 if (!lt.is_enabled()) { 365 return; 366 } 367 368 LogStream ls(lt); 369 370 // Print the population vector for the current epoch, and 371 // for the previous epoch, as well as the computed mortality 372 // ratio for each extant cohort. 373 const uint cur_epoch = _epoch; 374 const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1: markWord::max_age; 375 376 const AgeTable* cur_pv = _global_age_table[cur_epoch]; 377 const AgeTable* prev_pv = _global_age_table[prev_epoch]; 378 379 const uint tt = tenuring_threshold(); 380 381 size_t total= 0; 382 for (uint i = 1; i < MAX_COHORTS; i++) { 383 const size_t prev_pop = prev_pv->sizes[i-1]; // (i-1) OK because i >= 1 384 const size_t cur_pop = cur_pv->sizes[i]; 385 const double mr = mortality_rate(prev_pop, cur_pop); 386 // Suppress printing when everything is zero 387 if (prev_pop + cur_pop > 0) { 388 ls.print_cr(" - age %3u: prev " SIZE_FORMAT_W(10) " bytes, curr " SIZE_FORMAT_W(10) " bytes, mortality %.2f ", 389 i, prev_pop*oopSize, cur_pop*oopSize, mr); 390 } 391 total += cur_pop; 392 if (i == tt) { 393 // Underline the cohort for tenuring threshold (if < MAX_COHORTS) 394 ls.print_cr("----------------------------------------------------------------------------"); 395 } 396 } 397 CENSUS_NOISE(_global_noise[cur_epoch].print(ls, total);) 398 } 399 400 #ifdef SHENANDOAH_CENSUS_NOISE 401 void ShenandoahNoiseStats::print(LogStream& ls, const size_t total) { 402 if (total > 0) { 403 const float f_skipped = (float)skipped/(float)total; 404 const float f_aged = (float)aged/(float)total; 405 const float f_clamped = (float)clamped/(float)total; 406 const float f_young = (float)young/(float)total; 407 ls.print_cr("Skipped: " SIZE_FORMAT_W(10) " (%.2f), R-Aged: " SIZE_FORMAT_W(10) " (%.2f), " 408 "Clamped: " SIZE_FORMAT_W(10) " (%.2f), R-Young: " SIZE_FORMAT_W(10) " (%.2f)", 409 skipped*oopSize, f_skipped, aged*oopSize, f_aged, 410 clamped*oopSize, f_clamped, young*oopSize, f_young); 411 } 412 } 413 #endif // SHENANDOAH_CENSUS_NOISE