1 /* 2 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. 3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 28 #include "gc/shared/barrierSetNMethod.hpp" 29 #include "gc/shared/collectorCounters.hpp" 30 #include "gc/shared/continuationGCSupport.inline.hpp" 31 #include "gc/shenandoah/shenandoahBreakpoint.hpp" 32 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 33 #include "gc/shenandoah/shenandoahConcurrentGC.hpp" 34 #include "gc/shenandoah/shenandoahFreeSet.hpp" 35 #include "gc/shenandoah/shenandoahGeneration.hpp" 36 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 37 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 38 #include "gc/shenandoah/shenandoahLock.hpp" 39 #include "gc/shenandoah/shenandoahMark.inline.hpp" 40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 42 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 43 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 45 #include "gc/shenandoah/shenandoahStackWatermark.hpp" 46 #include "gc/shenandoah/shenandoahUtils.hpp" 47 #include "gc/shenandoah/shenandoahVerifier.hpp" 48 #include "gc/shenandoah/shenandoahVMOperations.hpp" 49 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 50 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 51 #include "memory/allocation.hpp" 52 #include "prims/jvmtiTagMap.hpp" 53 #include "runtime/vmThread.hpp" 54 #include "utilities/events.hpp" 55 56 // Breakpoint support 57 class ShenandoahBreakpointGCScope : public StackObj { 58 private: 59 const GCCause::Cause _cause; 60 public: 61 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) { 62 if (cause == GCCause::_wb_breakpoint) { 63 ShenandoahBreakpoint::start_gc(); 64 ShenandoahBreakpoint::at_before_gc(); 65 } 66 } 67 68 ~ShenandoahBreakpointGCScope() { 69 if (_cause == GCCause::_wb_breakpoint) { 70 ShenandoahBreakpoint::at_after_gc(); 71 } 72 } 73 }; 74 75 class ShenandoahBreakpointMarkScope : public StackObj { 76 private: 77 const GCCause::Cause _cause; 78 public: 79 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) { 80 if (_cause == GCCause::_wb_breakpoint) { 81 ShenandoahBreakpoint::at_after_marking_started(); 82 } 83 } 84 85 ~ShenandoahBreakpointMarkScope() { 86 if (_cause == GCCause::_wb_breakpoint) { 87 ShenandoahBreakpoint::at_before_marking_completed(); 88 } 89 } 90 }; 91 92 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : 93 _mark(generation), 94 _degen_point(ShenandoahDegenPoint::_degenerated_unset), 95 _abbreviated(false), 96 _do_old_gc_bootstrap(do_old_gc_bootstrap), 97 _generation(generation) { 98 } 99 100 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { 101 return _degen_point; 102 } 103 104 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { 105 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 106 heap->start_conc_gc(); 107 108 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause); 109 110 // Reset for upcoming marking 111 entry_reset(); 112 113 // Start initial mark under STW 114 vmop_entry_init_mark(); 115 116 { 117 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause); 118 119 // Reset task queue stats here, rather than in mark_concurrent_roots, 120 // because remembered set scan will `push` oops into the queues and 121 // resetting after this happens will lose those counts. 122 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); 123 124 // Concurrent remembered set scanning 125 entry_scan_remembered_set(); 126 // TODO: When RS scanning yields, we will need a check_cancellation_and_abort() degeneration point here. 127 128 // Concurrent mark roots 129 entry_mark_roots(); 130 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) return false; 131 132 // Continue concurrent mark 133 entry_mark(); 134 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false; 135 } 136 137 // Complete marking under STW, and start evacuation 138 vmop_entry_final_mark(); 139 140 // If GC was cancelled before final mark, then the safepoint operation will do nothing 141 // and the concurrent mark will still be in progress. In this case it is safe to resume 142 // the degenerated cycle from the marking phase. On the other hand, if the GC is cancelled 143 // after final mark (but before this check), then the final mark safepoint operation 144 // will have finished the mark (setting concurrent mark in progress to false). Final mark 145 // will also have setup state (in concurrent stack processing) that will not be safe to 146 // resume from the marking phase in the degenerated cycle. That is, if the cancellation 147 // occurred after final mark, we must resume the degenerated cycle after the marking phase. 148 if (_generation->is_concurrent_mark_in_progress() && check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { 149 assert(!heap->is_concurrent_weak_root_in_progress(), "Weak roots should not be in progress when concurrent mark is in progress"); 150 return false; 151 } 152 153 // Concurrent stack processing 154 if (heap->is_evacuation_in_progress()) { 155 entry_thread_roots(); 156 } 157 158 // Process weak roots that might still point to regions that would be broken by cleanup 159 if (heap->is_concurrent_weak_root_in_progress()) { 160 entry_weak_refs(); 161 entry_weak_roots(); 162 } 163 164 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim 165 // the space. This would be the last action if there is nothing to evacuate. Note that 166 // we will not age young-gen objects in the case that we skip evacuation. 167 entry_cleanup_early(); 168 169 { 170 ShenandoahHeapLocker locker(heap->lock()); 171 heap->free_set()->log_status(); 172 } 173 174 // Perform concurrent class unloading 175 if (heap->unload_classes() && 176 heap->is_concurrent_weak_root_in_progress()) { 177 entry_class_unloading(); 178 } 179 180 // Processing strong roots 181 // This may be skipped if there is nothing to update/evacuate. 182 // If so, strong_root_in_progress would be unset. 183 if (heap->is_concurrent_strong_root_in_progress()) { 184 entry_strong_roots(); 185 } 186 187 // Global marking has completed. We need to fill in any unmarked objects in the old generation 188 // so that subsequent remembered set scans will not walk pointers into reclaimed memory. 189 if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->is_global()) { 190 entry_global_coalesce_and_fill(); 191 } 192 193 // Continue the cycle with evacuation and optional update-refs. 194 // This may be skipped if there is nothing to evacuate. 195 // If so, evac_in_progress would be unset by collection set preparation code. 196 if (heap->is_evacuation_in_progress()) { 197 // Concurrently evacuate 198 entry_evacuate(); 199 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false; 200 201 // Perform update-refs phase. 202 vmop_entry_init_updaterefs(); 203 entry_updaterefs(); 204 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false; 205 206 // Concurrent update thread roots 207 entry_update_thread_roots(); 208 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false; 209 210 vmop_entry_final_updaterefs(); 211 212 // Update references freed up collection set, kick the cleanup to reclaim the space. 213 entry_cleanup_complete(); 214 } else { 215 // We chose not to evacuate because we found sufficient immediate garbage. 216 vmop_entry_final_roots(); 217 _abbreviated = true; 218 } 219 220 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an 221 // abbreviated cycle. 222 if (heap->mode()->is_generational()) { 223 bool success; 224 size_t region_xfer; 225 const char* region_destination; 226 ShenandoahYoungGeneration* young_gen = heap->young_generation(); 227 ShenandoahGeneration* old_gen = heap->old_generation(); 228 { 229 ShenandoahHeapLocker locker(heap->lock()); 230 231 size_t old_region_surplus = heap->get_old_region_surplus(); 232 size_t old_region_deficit = heap->get_old_region_deficit(); 233 if (old_region_surplus) { 234 success = heap->generation_sizer()->transfer_to_young(old_region_surplus); 235 region_destination = "young"; 236 region_xfer = old_region_surplus; 237 } else if (old_region_deficit) { 238 success = heap->generation_sizer()->transfer_to_old(old_region_deficit); 239 region_destination = "old"; 240 region_xfer = old_region_deficit; 241 if (!success) { 242 ((ShenandoahOldHeuristics *) old_gen->heuristics())->trigger_cannot_expand(); 243 } 244 } else { 245 region_destination = "none"; 246 region_xfer = 0; 247 success = true; 248 } 249 heap->set_old_region_surplus(0); 250 heap->set_old_region_deficit(0); 251 252 size_t old_usage_before_evac = heap->capture_old_usage(0); 253 size_t old_usage_now = old_gen->used(); 254 size_t promoted_bytes = old_usage_now - old_usage_before_evac; 255 heap->set_previous_promotion(promoted_bytes); 256 heap->set_young_evac_reserve(0); 257 heap->set_old_evac_reserve(0); 258 heap->reset_old_evac_expended(); 259 heap->set_promoted_reserve(0); 260 } 261 262 // Report outside the heap lock 263 size_t young_available = young_gen->available(); 264 size_t old_available = old_gen->available(); 265 log_info(gc, ergo)("After cleanup, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " 266 SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", 267 success? "successfully transferred": "failed to transfer", region_xfer, region_destination, 268 byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), 269 byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); 270 } 271 return true; 272 } 273 274 void ShenandoahConcurrentGC::vmop_entry_init_mark() { 275 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 276 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 277 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); 278 279 heap->try_inject_alloc_failure(); 280 VM_ShenandoahInitMark op(this); 281 VMThread::execute(&op); // jump to entry_init_mark() under safepoint 282 } 283 284 void ShenandoahConcurrentGC::vmop_entry_final_mark() { 285 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 286 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 287 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross); 288 289 heap->try_inject_alloc_failure(); 290 VM_ShenandoahFinalMarkStartEvac op(this); 291 VMThread::execute(&op); // jump to entry_final_mark under safepoint 292 } 293 294 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() { 295 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 296 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 297 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross); 298 299 heap->try_inject_alloc_failure(); 300 VM_ShenandoahInitUpdateRefs op(this); 301 VMThread::execute(&op); 302 } 303 304 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { 305 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 306 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 307 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross); 308 309 heap->try_inject_alloc_failure(); 310 VM_ShenandoahFinalUpdateRefs op(this); 311 VMThread::execute(&op); 312 } 313 314 void ShenandoahConcurrentGC::vmop_entry_final_roots() { 315 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 316 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 317 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); 318 319 // This phase does not use workers, no need for setup 320 heap->try_inject_alloc_failure(); 321 VM_ShenandoahFinalRoots op(this); 322 VMThread::execute(&op); 323 } 324 325 void ShenandoahConcurrentGC::entry_init_mark() { 326 const char* msg = init_mark_event_message(); 327 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); 328 EventMark em("%s", msg); 329 330 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 331 ShenandoahWorkerPolicy::calc_workers_for_init_marking(), 332 "init marking"); 333 334 op_init_mark(); 335 } 336 337 void ShenandoahConcurrentGC::entry_final_mark() { 338 const char* msg = final_mark_event_message(); 339 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); 340 EventMark em("%s", msg); 341 342 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 343 ShenandoahWorkerPolicy::calc_workers_for_final_marking(), 344 "final marking"); 345 346 op_final_mark(); 347 } 348 349 void ShenandoahConcurrentGC::entry_init_updaterefs() { 350 static const char* msg = "Pause Init Update Refs"; 351 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs); 352 EventMark em("%s", msg); 353 354 // No workers used in this phase, no setup required 355 op_init_updaterefs(); 356 } 357 358 void ShenandoahConcurrentGC::entry_final_updaterefs() { 359 static const char* msg = "Pause Final Update Refs"; 360 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs); 361 EventMark em("%s", msg); 362 363 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 364 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(), 365 "final reference update"); 366 367 op_final_updaterefs(); 368 } 369 370 void ShenandoahConcurrentGC::entry_final_roots() { 371 static const char* msg = "Pause Final Roots"; 372 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots); 373 EventMark em("%s", msg); 374 375 op_final_roots(); 376 } 377 378 void ShenandoahConcurrentGC::entry_reset() { 379 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 380 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 381 static const char* msg = "Concurrent reset"; 382 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset); 383 EventMark em("%s", msg); 384 385 ShenandoahWorkerScope scope(heap->workers(), 386 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 387 "concurrent reset"); 388 389 heap->try_inject_alloc_failure(); 390 op_reset(); 391 } 392 393 void ShenandoahConcurrentGC::entry_scan_remembered_set() { 394 if (_generation->is_young()) { 395 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 396 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 397 const char* msg = "Concurrent remembered set scanning"; 398 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset); 399 EventMark em("%s", msg); 400 401 ShenandoahWorkerScope scope(heap->workers(), 402 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(), 403 msg); 404 405 heap->try_inject_alloc_failure(); 406 _generation->scan_remembered_set(true /* is_concurrent */); 407 } 408 } 409 410 void ShenandoahConcurrentGC::entry_mark_roots() { 411 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 412 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 413 const char* msg = "Concurrent marking roots"; 414 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots); 415 EventMark em("%s", msg); 416 417 ShenandoahWorkerScope scope(heap->workers(), 418 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 419 "concurrent marking roots"); 420 421 heap->try_inject_alloc_failure(); 422 op_mark_roots(); 423 } 424 425 void ShenandoahConcurrentGC::entry_mark() { 426 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 427 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 428 const char* msg = conc_mark_event_message(); 429 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark); 430 EventMark em("%s", msg); 431 432 ShenandoahWorkerScope scope(heap->workers(), 433 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 434 "concurrent marking"); 435 436 heap->try_inject_alloc_failure(); 437 op_mark(); 438 } 439 440 void ShenandoahConcurrentGC::entry_thread_roots() { 441 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 442 static const char* msg = "Concurrent thread roots"; 443 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots); 444 EventMark em("%s", msg); 445 446 ShenandoahWorkerScope scope(heap->workers(), 447 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 448 msg); 449 450 heap->try_inject_alloc_failure(); 451 op_thread_roots(); 452 } 453 454 void ShenandoahConcurrentGC::entry_weak_refs() { 455 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 456 static const char* msg = "Concurrent weak references"; 457 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs); 458 EventMark em("%s", msg); 459 460 ShenandoahWorkerScope scope(heap->workers(), 461 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(), 462 "concurrent weak references"); 463 464 heap->try_inject_alloc_failure(); 465 op_weak_refs(); 466 } 467 468 void ShenandoahConcurrentGC::entry_weak_roots() { 469 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 470 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 471 static const char* msg = "Concurrent weak roots"; 472 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots); 473 EventMark em("%s", msg); 474 475 ShenandoahWorkerScope scope(heap->workers(), 476 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 477 "concurrent weak root"); 478 479 heap->try_inject_alloc_failure(); 480 op_weak_roots(); 481 } 482 483 void ShenandoahConcurrentGC::entry_class_unloading() { 484 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 485 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 486 static const char* msg = "Concurrent class unloading"; 487 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload); 488 EventMark em("%s", msg); 489 490 ShenandoahWorkerScope scope(heap->workers(), 491 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 492 "concurrent class unloading"); 493 494 heap->try_inject_alloc_failure(); 495 op_class_unloading(); 496 } 497 498 void ShenandoahConcurrentGC::entry_strong_roots() { 499 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 500 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 501 static const char* msg = "Concurrent strong roots"; 502 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots); 503 EventMark em("%s", msg); 504 505 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots); 506 507 ShenandoahWorkerScope scope(heap->workers(), 508 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 509 "concurrent strong root"); 510 511 heap->try_inject_alloc_failure(); 512 op_strong_roots(); 513 } 514 515 void ShenandoahConcurrentGC::entry_cleanup_early() { 516 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 517 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 518 static const char* msg = "Concurrent cleanup"; 519 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */); 520 EventMark em("%s", msg); 521 522 // This phase does not use workers, no need for setup 523 heap->try_inject_alloc_failure(); 524 op_cleanup_early(); 525 } 526 527 void ShenandoahConcurrentGC::entry_evacuate() { 528 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 529 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 530 531 static const char* msg = "Concurrent evacuation"; 532 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac); 533 EventMark em("%s", msg); 534 535 ShenandoahWorkerScope scope(heap->workers(), 536 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), 537 "concurrent evacuation"); 538 539 heap->try_inject_alloc_failure(); 540 op_evacuate(); 541 } 542 543 void ShenandoahConcurrentGC::entry_update_thread_roots() { 544 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 545 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 546 547 static const char* msg = "Concurrent update thread roots"; 548 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots); 549 EventMark em("%s", msg); 550 551 // No workers used in this phase, no setup required 552 heap->try_inject_alloc_failure(); 553 op_update_thread_roots(); 554 } 555 556 void ShenandoahConcurrentGC::entry_updaterefs() { 557 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 558 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 559 static const char* msg = "Concurrent update references"; 560 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs); 561 EventMark em("%s", msg); 562 563 ShenandoahWorkerScope scope(heap->workers(), 564 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(), 565 "concurrent reference update"); 566 567 heap->try_inject_alloc_failure(); 568 op_updaterefs(); 569 } 570 571 void ShenandoahConcurrentGC::entry_cleanup_complete() { 572 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 573 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 574 static const char* msg = "Concurrent cleanup"; 575 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */); 576 EventMark em("%s", msg); 577 578 // This phase does not use workers, no need for setup 579 heap->try_inject_alloc_failure(); 580 op_cleanup_complete(); 581 } 582 583 void ShenandoahConcurrentGC::entry_global_coalesce_and_fill() { 584 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 585 586 const char* msg = "Coalescing and filling old regions in global collect"; 587 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); 588 589 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 590 EventMark em("%s", msg); 591 ShenandoahWorkerScope scope(heap->workers(), 592 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 593 "concurrent coalesce and fill"); 594 595 op_global_coalesce_and_fill(); 596 } 597 598 void ShenandoahConcurrentGC::op_reset() { 599 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 600 if (ShenandoahPacing) { 601 heap->pacer()->setup_for_reset(); 602 } 603 _generation->prepare_gc(); 604 } 605 606 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { 607 private: 608 ShenandoahMarkingContext* const _ctx; 609 public: 610 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} 611 612 void heap_region_do(ShenandoahHeapRegion* r) { 613 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); 614 if (r->is_active()) { 615 // Check if region needs updating its TAMS. We have updated it already during concurrent 616 // reset, so it is very likely we don't need to do another write here. Since most regions 617 // are not "active", this path is relatively rare. 618 if (_ctx->top_at_mark_start(r) != r->top()) { 619 _ctx->capture_top_at_mark_start(r); 620 } 621 } else { 622 assert(_ctx->top_at_mark_start(r) == r->top(), 623 "Region " SIZE_FORMAT " should already have correct TAMS", r->index()); 624 } 625 } 626 627 bool is_thread_safe() { return true; } 628 }; 629 630 void ShenandoahConcurrentGC::start_mark() { 631 _mark.start_mark(); 632 } 633 634 void ShenandoahConcurrentGC::op_init_mark() { 635 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 636 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 637 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); 638 639 assert(_generation->is_bitmap_clear(), "need clear marking bitmap"); 640 assert(!_generation->is_mark_complete(), "should not be complete"); 641 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 642 643 644 if (heap->mode()->is_generational()) { 645 if (_generation->is_young() || (_generation->is_global() && ShenandoahVerify)) { 646 // The current implementation of swap_remembered_set() copies the write-card-table 647 // to the read-card-table. The remembered sets are also swapped for GLOBAL collections 648 // so that the verifier works with the correct copy of the card table when verifying. 649 // TODO: This path should not really depend on ShenandoahVerify. 650 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); 651 _generation->swap_remembered_set(); 652 } 653 654 if (_generation->is_global()) { 655 heap->cancel_old_gc(); 656 } else if (heap->is_concurrent_old_mark_in_progress()) { 657 // Purge the SATB buffers, transferring any valid, old pointers to the 658 // old generation mark queue. Any pointers in a young region will be 659 // abandoned. 660 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb); 661 heap->transfer_old_pointers_from_satb(); 662 } 663 } 664 665 if (ShenandoahVerify) { 666 heap->verifier()->verify_before_concmark(); 667 } 668 669 if (VerifyBeforeGC) { 670 Universe::verify(); 671 } 672 673 _generation->set_concurrent_mark_in_progress(true); 674 675 start_mark(); 676 677 if (_do_old_gc_bootstrap) { 678 // Update region state for both young and old regions 679 // TODO: We should be able to pull this out of the safepoint for the bootstrap 680 // cycle. The top of an old region will only move when a GC cycle evacuates 681 // objects into it. When we start an old cycle, we know that nothing can touch 682 // the top of old regions. 683 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 684 ShenandoahInitMarkUpdateRegionStateClosure cl; 685 heap->parallel_heap_region_iterate(&cl); 686 } else { 687 // Update region state for only young regions 688 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 689 ShenandoahInitMarkUpdateRegionStateClosure cl; 690 _generation->parallel_heap_region_iterate(&cl); 691 } 692 693 // Weak reference processing 694 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 695 rp->reset_thread_locals(); 696 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs()); 697 698 // Make above changes visible to worker threads 699 OrderAccess::fence(); 700 701 // Arm nmethods for concurrent marking. When a nmethod is about to be executed, 702 // we need to make sure that all its metadata are marked. alternative is to remark 703 // thread roots at final mark pause, but it can be potential latency killer. 704 if (heap->unload_classes()) { 705 ShenandoahCodeRoots::arm_nmethods(); 706 } 707 708 ShenandoahStackWatermark::change_epoch_id(); 709 if (ShenandoahPacing) { 710 heap->pacer()->setup_for_mark(); 711 } 712 } 713 714 void ShenandoahConcurrentGC::op_mark_roots() { 715 _mark.mark_concurrent_roots(); 716 } 717 718 void ShenandoahConcurrentGC::op_mark() { 719 _mark.concurrent_mark(); 720 } 721 722 void ShenandoahConcurrentGC::op_final_mark() { 723 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 724 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 725 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 726 727 if (ShenandoahVerify) { 728 heap->verifier()->verify_roots_no_forwarded(); 729 } 730 731 if (!heap->cancelled_gc()) { 732 _mark.finish_mark(); 733 assert(!heap->cancelled_gc(), "STW mark cannot OOM"); 734 735 // Notify JVMTI that the tagmap table will need cleaning. 736 JvmtiTagMap::set_needs_cleaning(); 737 738 // The collection set is chosen by prepare_regions_and_collection_set(). 739 // 740 // TODO: Under severe memory overload conditions that can be checked here, we may want to limit 741 // the inclusion of old-gen candidates within the collection set. This would allow us to prioritize efforts on 742 // evacuating young-gen, This remediation is most appropriate when old-gen availability is very high (so there 743 // are negligible negative impacts from delaying completion of old-gen evacuation) and when young-gen collections 744 // are "under duress" (as signalled by very low availability of memory within young-gen, indicating that/ young-gen 745 // collections are not triggering frequently enough). 746 _generation->prepare_regions_and_collection_set(true /*concurrent*/); 747 748 // Upon return from prepare_regions_and_collection_set(), certain parameters have been established to govern the 749 // evacuation efforts that are about to begin. In particular: 750 // 751 // heap->get_promoted_reserve() represents the amount of memory within old-gen's available memory that has 752 // been set aside to hold objects promoted from young-gen memory. This represents an estimated percentage 753 // of the live young-gen memory within the collection set. If there is more data ready to be promoted than 754 // can fit within this reserve, the promotion of some objects will be deferred until a subsequent evacuation 755 // pass. 756 // 757 // heap->get_old_evac_reserve() represents the amount of memory within old-gen's available memory that has been 758 // set aside to hold objects evacuated from the old-gen collection set. 759 // 760 // heap->get_young_evac_reserve() represents the amount of memory within young-gen's available memory that has 761 // been set aside to hold objects evacuated from the young-gen collection set. Conservatively, this value 762 // equals the entire amount of live young-gen memory within the collection set, even though some of this memory 763 // will likely be promoted. 764 765 // Has to be done after cset selection 766 heap->prepare_concurrent_roots(); 767 768 if (heap->mode()->is_generational()) { 769 ShenandoahGeneration* young_gen = heap->young_generation(); 770 size_t humongous_regions_promoted = heap->get_promotable_humongous_regions(); 771 size_t regular_regions_promoted_in_place = heap->get_regular_regions_promoted_in_place(); 772 if (!heap->collection_set()->is_empty() || (humongous_regions_promoted + regular_regions_promoted_in_place > 0)) { 773 // Even if the collection set is empty, we need to do evacuation if there are regions to be promoted in place. 774 // Concurrent evacuation takes responsibility for registering objects and setting the remembered set cards to dirty. 775 776 LogTarget(Debug, gc, cset) lt; 777 if (lt.is_enabled()) { 778 ResourceMark rm; 779 LogStream ls(lt); 780 heap->collection_set()->print_on(&ls); 781 } 782 783 if (ShenandoahVerify) { 784 heap->verifier()->verify_before_evacuation(); 785 } 786 // TODO: we do not need to run update-references following evacuation if collection_set->is_empty(). 787 788 heap->set_evacuation_in_progress(true); 789 // From here on, we need to update references. 790 heap->set_has_forwarded_objects(true); 791 792 // Verify before arming for concurrent processing. 793 // Otherwise, verification can trigger stack processing. 794 if (ShenandoahVerify) { 795 heap->verifier()->verify_during_evacuation(); 796 } 797 798 // Arm nmethods/stack for concurrent processing 799 ShenandoahCodeRoots::arm_nmethods(); 800 ShenandoahStackWatermark::change_epoch_id(); 801 802 if (ShenandoahPacing) { 803 heap->pacer()->setup_for_evac(); 804 } 805 } else { 806 if (ShenandoahVerify) { 807 heap->verifier()->verify_after_concmark(); 808 } 809 810 if (VerifyAfterGC) { 811 Universe::verify(); 812 } 813 } 814 } else { 815 // Not is_generational() 816 if (!heap->collection_set()->is_empty()) { 817 LogTarget(Info, gc, ergo) lt; 818 if (lt.is_enabled()) { 819 ResourceMark rm; 820 LogStream ls(lt); 821 heap->collection_set()->print_on(&ls); 822 } 823 824 if (ShenandoahVerify) { 825 heap->verifier()->verify_before_evacuation(); 826 } 827 828 heap->set_evacuation_in_progress(true); 829 // From here on, we need to update references. 830 heap->set_has_forwarded_objects(true); 831 832 // Verify before arming for concurrent processing. 833 // Otherwise, verification can trigger stack processing. 834 if (ShenandoahVerify) { 835 heap->verifier()->verify_during_evacuation(); 836 } 837 838 // Arm nmethods/stack for concurrent processing 839 ShenandoahCodeRoots::arm_nmethods(); 840 ShenandoahStackWatermark::change_epoch_id(); 841 842 if (ShenandoahPacing) { 843 heap->pacer()->setup_for_evac(); 844 } 845 } else { 846 if (ShenandoahVerify) { 847 heap->verifier()->verify_after_concmark(); 848 } 849 850 if (VerifyAfterGC) { 851 Universe::verify(); 852 } 853 } 854 } 855 } 856 } 857 858 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure { 859 private: 860 OopClosure* const _oops; 861 862 public: 863 ShenandoahConcurrentEvacThreadClosure(OopClosure* oops); 864 void do_thread(Thread* thread); 865 }; 866 867 ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : 868 _oops(oops) { 869 } 870 871 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) { 872 JavaThread* const jt = JavaThread::cast(thread); 873 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc); 874 ShenandoahThreadLocalData::enable_plab_promotions(thread); 875 } 876 877 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask { 878 private: 879 ShenandoahJavaThreadsIterator _java_threads; 880 881 public: 882 ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) : 883 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"), 884 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) { 885 } 886 887 void work(uint worker_id) { 888 Thread* worker_thread = Thread::current(); 889 ShenandoahThreadLocalData::enable_plab_promotions(worker_thread); 890 891 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure. 892 // Otherwise, may deadlock with watermark lock 893 ShenandoahContextEvacuateUpdateRootsClosure oops_cl; 894 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl); 895 _java_threads.threads_do(&thr_cl, worker_id); 896 } 897 }; 898 899 void ShenandoahConcurrentGC::op_thread_roots() { 900 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 901 assert(heap->is_evacuation_in_progress(), "Checked by caller"); 902 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots); 903 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers()); 904 heap->workers()->run_task(&task); 905 } 906 907 void ShenandoahConcurrentGC::op_weak_refs() { 908 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 909 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 910 // Concurrent weak refs processing 911 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs); 912 if (heap->gc_cause() == GCCause::_wb_breakpoint) { 913 ShenandoahBreakpoint::at_after_reference_processing_started(); 914 } 915 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */); 916 } 917 918 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure { 919 private: 920 ShenandoahHeap* const _heap; 921 ShenandoahMarkingContext* const _mark_context; 922 bool _evac_in_progress; 923 Thread* const _thread; 924 925 public: 926 ShenandoahEvacUpdateCleanupOopStorageRootsClosure(); 927 void do_oop(oop* p); 928 void do_oop(narrowOop* p); 929 }; 930 931 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() : 932 _heap(ShenandoahHeap::heap()), 933 _mark_context(ShenandoahHeap::heap()->marking_context()), 934 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()), 935 _thread(Thread::current()) { 936 } 937 938 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { 939 const oop obj = RawAccess<>::oop_load(p); 940 if (!CompressedOops::is_null(obj)) { 941 if (!_mark_context->is_marked(obj)) { 942 if (_heap->is_in_active_generation(obj)) { 943 // TODO: This worries me. Here we are asserting that an unmarked from-space object is 'correct'. 944 // Normally, I would call this a bogus assert, but there seems to be a legitimate use-case for 945 // accessing from-space objects during class unloading. However, the from-space object may have 946 // been "filled". We've made no effort to prevent old generation classes being unloaded by young 947 // gen (and vice-versa). 948 shenandoah_assert_correct(p, obj); 949 ShenandoahHeap::atomic_clear_oop(p, obj); 950 } 951 } else if (_evac_in_progress && _heap->in_collection_set(obj)) { 952 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 953 if (resolved == obj) { 954 resolved = _heap->evacuate_object(obj, _thread); 955 } 956 ShenandoahHeap::atomic_update_oop(resolved, p, obj); 957 assert(_heap->cancelled_gc() || 958 _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved), 959 "Sanity"); 960 } 961 } 962 } 963 964 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) { 965 ShouldNotReachHere(); 966 } 967 968 class ShenandoahIsCLDAliveClosure : public CLDClosure { 969 public: 970 void do_cld(ClassLoaderData* cld) { 971 cld->is_alive(); 972 } 973 }; 974 975 class ShenandoahIsNMethodAliveClosure: public NMethodClosure { 976 public: 977 void do_nmethod(nmethod* n) { 978 n->is_unloading(); 979 } 980 }; 981 982 // This task not only evacuates/updates marked weak roots, but also "null" 983 // dead weak roots. 984 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask { 985 private: 986 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots; 987 988 // Roots related to concurrent class unloading 989 ShenandoahClassLoaderDataRoots<true /* concurrent */> 990 _cld_roots; 991 ShenandoahConcurrentNMethodIterator _nmethod_itr; 992 ShenandoahPhaseTimings::Phase _phase; 993 994 public: 995 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 996 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"), 997 _vm_roots(phase), 998 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 999 _nmethod_itr(ShenandoahCodeRoots::table()), 1000 _phase(phase) { 1001 if (ShenandoahHeap::heap()->unload_classes()) { 1002 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); 1003 _nmethod_itr.nmethods_do_begin(); 1004 } 1005 } 1006 1007 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() { 1008 if (ShenandoahHeap::heap()->unload_classes()) { 1009 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); 1010 _nmethod_itr.nmethods_do_end(); 1011 } 1012 // Notify runtime data structures of potentially dead oops 1013 _vm_roots.report_num_dead(); 1014 } 1015 1016 void work(uint worker_id) { 1017 ShenandoahConcurrentWorkerSession worker_session(worker_id); 1018 ShenandoahSuspendibleThreadSetJoiner sts_join; 1019 { 1020 ShenandoahEvacOOMScope oom; 1021 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration 1022 // may race against OopStorage::release() calls. 1023 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl; 1024 _vm_roots.oops_do(&cl, worker_id); 1025 } 1026 1027 // If we are going to perform concurrent class unloading later on, we need to 1028 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we 1029 // can cleanup immediate garbage sooner. 1030 if (ShenandoahHeap::heap()->unload_classes()) { 1031 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the 1032 // CLD's holder or evacuate it. 1033 { 1034 ShenandoahIsCLDAliveClosure is_cld_alive; 1035 _cld_roots.cld_do(&is_cld_alive, worker_id); 1036 } 1037 1038 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods. 1039 // The closure calls nmethod->is_unloading(). The is_unloading 1040 // state is cached, therefore, during concurrent class unloading phase, 1041 // we will not touch the metadata of unloading nmethods 1042 { 1043 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 1044 ShenandoahIsNMethodAliveClosure is_nmethod_alive; 1045 _nmethod_itr.nmethods_do(&is_nmethod_alive); 1046 } 1047 } 1048 } 1049 }; 1050 1051 void ShenandoahConcurrentGC::op_weak_roots() { 1052 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1053 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 1054 // Concurrent weak root processing 1055 { 1056 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work); 1057 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work); 1058 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work); 1059 heap->workers()->run_task(&task); 1060 } 1061 1062 // Perform handshake to flush out dead oops 1063 { 1064 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous); 1065 heap->rendezvous_threads(); 1066 } 1067 } 1068 1069 void ShenandoahConcurrentGC::op_class_unloading() { 1070 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1071 assert (heap->is_concurrent_weak_root_in_progress() && 1072 heap->unload_classes(), 1073 "Checked by caller"); 1074 heap->do_class_unloading(); 1075 } 1076 1077 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure { 1078 private: 1079 BarrierSetNMethod* const _bs; 1080 ShenandoahEvacuateUpdateMetadataClosure _cl; 1081 1082 public: 1083 ShenandoahEvacUpdateCodeCacheClosure() : 1084 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()), 1085 _cl() { 1086 } 1087 1088 void do_nmethod(nmethod* n) { 1089 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n); 1090 ShenandoahReentrantLocker locker(data->lock()); 1091 // Setup EvacOOM scope below reentrant lock to avoid deadlock with 1092 // nmethod_entry_barrier 1093 ShenandoahEvacOOMScope oom; 1094 data->oops_do(&_cl, true/*fix relocation*/); 1095 _bs->disarm(n); 1096 } 1097 }; 1098 1099 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask { 1100 private: 1101 ShenandoahPhaseTimings::Phase _phase; 1102 ShenandoahVMRoots<true /*concurrent*/> _vm_roots; 1103 ShenandoahClassLoaderDataRoots<true /*concurrent*/> 1104 _cld_roots; 1105 ShenandoahConcurrentNMethodIterator _nmethod_itr; 1106 1107 public: 1108 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 1109 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"), 1110 _phase(phase), 1111 _vm_roots(phase), 1112 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 1113 _nmethod_itr(ShenandoahCodeRoots::table()) { 1114 if (!ShenandoahHeap::heap()->unload_classes()) { 1115 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); 1116 _nmethod_itr.nmethods_do_begin(); 1117 } 1118 } 1119 1120 ~ShenandoahConcurrentRootsEvacUpdateTask() { 1121 if (!ShenandoahHeap::heap()->unload_classes()) { 1122 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); 1123 _nmethod_itr.nmethods_do_end(); 1124 } 1125 } 1126 1127 void work(uint worker_id) { 1128 ShenandoahConcurrentWorkerSession worker_session(worker_id); 1129 { 1130 ShenandoahEvacOOMScope oom; 1131 { 1132 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration 1133 // may race against OopStorage::release() calls. 1134 ShenandoahContextEvacuateUpdateRootsClosure cl; 1135 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id); 1136 } 1137 1138 { 1139 ShenandoahEvacuateUpdateMetadataClosure cl; 1140 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong); 1141 _cld_roots.cld_do(&clds, worker_id); 1142 } 1143 } 1144 1145 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier. 1146 if (!ShenandoahHeap::heap()->unload_classes()) { 1147 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 1148 ShenandoahEvacUpdateCodeCacheClosure cl; 1149 _nmethod_itr.nmethods_do(&cl); 1150 } 1151 } 1152 }; 1153 1154 void ShenandoahConcurrentGC::op_strong_roots() { 1155 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1156 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller"); 1157 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots); 1158 heap->workers()->run_task(&task); 1159 heap->set_concurrent_strong_root_in_progress(false); 1160 } 1161 1162 void ShenandoahConcurrentGC::op_cleanup_early() { 1163 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1164 } 1165 1166 void ShenandoahConcurrentGC::op_evacuate() { 1167 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/); 1168 } 1169 1170 void ShenandoahConcurrentGC::op_init_updaterefs() { 1171 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1172 heap->set_evacuation_in_progress(false); 1173 heap->set_concurrent_weak_root_in_progress(false); 1174 heap->prepare_update_heap_references(true /*concurrent*/); 1175 heap->set_update_refs_in_progress(true); 1176 if (ShenandoahVerify) { 1177 heap->verifier()->verify_before_updaterefs(); 1178 } 1179 if (ShenandoahPacing) { 1180 heap->pacer()->setup_for_updaterefs(); 1181 } 1182 } 1183 1184 void ShenandoahConcurrentGC::op_updaterefs() { 1185 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/); 1186 } 1187 1188 class ShenandoahUpdateThreadClosure : public HandshakeClosure { 1189 private: 1190 ShenandoahUpdateRefsClosure _cl; 1191 public: 1192 ShenandoahUpdateThreadClosure(); 1193 void do_thread(Thread* thread); 1194 }; 1195 1196 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() : 1197 HandshakeClosure("Shenandoah Update Thread Roots") { 1198 } 1199 1200 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) { 1201 if (thread->is_Java_thread()) { 1202 JavaThread* jt = JavaThread::cast(thread); 1203 ResourceMark rm; 1204 jt->oops_do(&_cl, nullptr); 1205 } 1206 } 1207 1208 void ShenandoahConcurrentGC::op_update_thread_roots() { 1209 ShenandoahUpdateThreadClosure cl; 1210 Handshake::execute(&cl); 1211 } 1212 1213 void ShenandoahConcurrentGC::op_final_updaterefs() { 1214 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1215 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); 1216 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references"); 1217 1218 heap->finish_concurrent_roots(); 1219 1220 // Clear cancelled GC, if set. On cancellation path, the block before would handle 1221 // everything. 1222 if (heap->cancelled_gc()) { 1223 heap->clear_cancelled_gc(true /* clear oom handler */); 1224 } 1225 1226 // Has to be done before cset is clear 1227 if (ShenandoahVerify) { 1228 heap->verifier()->verify_roots_in_to_space(); 1229 } 1230 1231 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) { 1232 // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to 1233 // objects in the collection set. After those objects are evacuated, the pointers in the 1234 // SATB are no longer safe. Once we have finished update references, we are guaranteed that 1235 // no more writes to the collection set are possible. 1236 // 1237 // This will transfer any old pointers in _active_ regions from the SATB to the old gen 1238 // mark queues. All other pointers will be discarded. This would also discard any pointers 1239 // in old regions that were included in a mixed evacuation. We aren't using the SATB filter 1240 // methods here because we cannot control when they execute. If the SATB filter runs _after_ 1241 // a region has been recycled, we will not be able to detect the bad pointer. 1242 // 1243 // We are not concerned about skipping this step in abbreviated cycles because regions 1244 // with no live objects cannot have been written to and so cannot have entries in the SATB 1245 // buffers. 1246 heap->transfer_old_pointers_from_satb(); 1247 } 1248 1249 heap->update_heap_region_states(true /*concurrent*/); 1250 1251 heap->set_update_refs_in_progress(false); 1252 heap->set_has_forwarded_objects(false); 1253 1254 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for 1255 // entire regions. Both of these relevant operations occur before final update refs. 1256 heap->set_aging_cycle(false); 1257 1258 if (ShenandoahVerify) { 1259 heap->verifier()->verify_after_updaterefs(); 1260 } 1261 1262 if (VerifyAfterGC) { 1263 Universe::verify(); 1264 } 1265 1266 heap->rebuild_free_set(true /*concurrent*/); 1267 } 1268 1269 void ShenandoahConcurrentGC::op_final_roots() { 1270 1271 ShenandoahHeap* heap = ShenandoahHeap::heap(); 1272 if (heap->is_aging_cycle()) { 1273 ShenandoahMarkingContext* ctx = heap->complete_marking_context(); 1274 1275 for (size_t i = 0; i < heap->num_regions(); i++) { 1276 ShenandoahHeapRegion *r = heap->get_region(i); 1277 if (r->is_active() && r->is_young()) { 1278 HeapWord* tams = ctx->top_at_mark_start(r); 1279 HeapWord* top = r->top(); 1280 if (top > tams) { 1281 r->reset_age(); 1282 } else { 1283 r->increment_age(); 1284 } 1285 } 1286 } 1287 } 1288 1289 ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false); 1290 } 1291 1292 void ShenandoahConcurrentGC::op_cleanup_complete() { 1293 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1294 } 1295 1296 void ShenandoahConcurrentGC::op_global_coalesce_and_fill() { 1297 ShenandoahHeap::heap()->coalesce_and_fill_old_regions(); 1298 } 1299 1300 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) { 1301 if (ShenandoahHeap::heap()->cancelled_gc()) { 1302 _degen_point = point; 1303 return true; 1304 } 1305 return false; 1306 } 1307 1308 const char* ShenandoahConcurrentGC::init_mark_event_message() const { 1309 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1310 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); 1311 if (heap->unload_classes()) { 1312 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Init Mark", " (unload classes)"); 1313 } else { 1314 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Init Mark", ""); 1315 } 1316 } 1317 1318 const char* ShenandoahConcurrentGC::final_mark_event_message() const { 1319 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1320 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1321 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running"); 1322 1323 if (heap->unload_classes()) { 1324 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Final Mark", " (unload classes)"); 1325 } else { 1326 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Final Mark", ""); 1327 } 1328 } 1329 1330 const char* ShenandoahConcurrentGC::conc_mark_event_message() const { 1331 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1332 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1333 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running"); 1334 if (heap->unload_classes()) { 1335 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Concurrent marking", " (unload classes)"); 1336 } else { 1337 SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Concurrent marking", ""); 1338 } 1339 }