1 /* 2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 29 #include "gc/shared/barrierSetNMethod.hpp" 30 #include "gc/shared/collectorCounters.hpp" 31 #include "gc/shared/continuationGCSupport.inline.hpp" 32 #include "gc/shenandoah/shenandoahBreakpoint.hpp" 33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp" 35 #include "gc/shenandoah/shenandoahFreeSet.hpp" 36 #include "gc/shenandoah/shenandoahGeneration.hpp" 37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 38 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 40 #include "gc/shenandoah/shenandoahLock.hpp" 41 #include "gc/shenandoah/shenandoahMark.inline.hpp" 42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 45 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 46 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 47 #include "gc/shenandoah/shenandoahStackWatermark.hpp" 48 #include "gc/shenandoah/shenandoahUtils.hpp" 49 #include "gc/shenandoah/shenandoahVerifier.hpp" 50 #include "gc/shenandoah/shenandoahVMOperations.hpp" 51 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 52 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 53 #include "memory/allocation.hpp" 54 #include "prims/jvmtiTagMap.hpp" 55 #include "runtime/vmThread.hpp" 56 #include "utilities/events.hpp" 57 58 // Breakpoint support 59 class ShenandoahBreakpointGCScope : public StackObj { 60 private: 61 const GCCause::Cause _cause; 62 public: 63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) { 64 if (cause == GCCause::_wb_breakpoint) { 65 ShenandoahBreakpoint::start_gc(); 66 ShenandoahBreakpoint::at_before_gc(); 67 } 68 } 69 70 ~ShenandoahBreakpointGCScope() { 71 if (_cause == GCCause::_wb_breakpoint) { 72 ShenandoahBreakpoint::at_after_gc(); 73 } 74 } 75 }; 76 77 class ShenandoahBreakpointMarkScope : public StackObj { 78 private: 79 const GCCause::Cause _cause; 80 public: 81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) { 82 if (_cause == GCCause::_wb_breakpoint) { 83 ShenandoahBreakpoint::at_after_marking_started(); 84 } 85 } 86 87 ~ShenandoahBreakpointMarkScope() { 88 if (_cause == GCCause::_wb_breakpoint) { 89 ShenandoahBreakpoint::at_before_marking_completed(); 90 } 91 } 92 }; 93 94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : 95 _mark(generation), 96 _degen_point(ShenandoahDegenPoint::_degenerated_unset), 97 _abbreviated(false), 98 _do_old_gc_bootstrap(do_old_gc_bootstrap), 99 _generation(generation) { 100 } 101 102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { 103 return _degen_point; 104 } 105 106 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { 107 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 108 109 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause); 110 111 // Reset for upcoming marking 112 entry_reset(); 113 114 // Start initial mark under STW 115 vmop_entry_init_mark(); 116 117 { 118 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause); 119 120 // Reset task queue stats here, rather than in mark_concurrent_roots, 121 // because remembered set scan will `push` oops into the queues and 122 // resetting after this happens will lose those counts. 123 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); 124 125 // Concurrent remembered set scanning 126 entry_scan_remembered_set(); 127 128 // Concurrent mark roots 129 entry_mark_roots(); 130 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) { 131 return false; 132 } 133 134 // Continue concurrent mark 135 entry_mark(); 136 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { 137 return false; 138 } 139 } 140 141 // Complete marking under STW, and start evacuation 142 vmop_entry_final_mark(); 143 144 // If the GC was cancelled before final mark, nothing happens on the safepoint. We are still 145 // in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled 146 // after final mark, then we've entered the evacuation phase and must resume the degenerated cycle 147 // from that phase. 148 if (_generation->is_concurrent_mark_in_progress()) { 149 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark); 150 assert(cancelled, "GC must have been cancelled between concurrent and final mark"); 151 return false; 152 } 153 154 // Concurrent stack processing 155 if (heap->is_evacuation_in_progress()) { 156 entry_thread_roots(); 157 } 158 159 // Process weak roots that might still point to regions that would be broken by cleanup 160 if (heap->is_concurrent_weak_root_in_progress()) { 161 entry_weak_refs(); 162 entry_weak_roots(); 163 } 164 165 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim 166 // the space. This would be the last action if there is nothing to evacuate. Note that 167 // we will not age young-gen objects in the case that we skip evacuation. 168 entry_cleanup_early(); 169 170 heap->free_set()->log_status_under_lock(); 171 172 // Perform concurrent class unloading 173 if (heap->unload_classes() && 174 heap->is_concurrent_weak_root_in_progress()) { 175 entry_class_unloading(); 176 } 177 178 // Processing strong roots 179 // This may be skipped if there is nothing to update/evacuate. 180 // If so, strong_root_in_progress would be unset. 181 if (heap->is_concurrent_strong_root_in_progress()) { 182 entry_strong_roots(); 183 } 184 185 // Continue the cycle with evacuation and optional update-refs. 186 // This may be skipped if there is nothing to evacuate. 187 // If so, evac_in_progress would be unset by collection set preparation code. 188 if (heap->is_evacuation_in_progress()) { 189 // Concurrently evacuate 190 entry_evacuate(); 191 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { 192 return false; 193 } 194 195 // Perform update-refs phase. 196 vmop_entry_init_updaterefs(); 197 entry_updaterefs(); 198 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) { 199 return false; 200 } 201 202 // Concurrent update thread roots 203 entry_update_thread_roots(); 204 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) { 205 return false; 206 } 207 208 vmop_entry_final_updaterefs(); 209 210 // Update references freed up collection set, kick the cleanup to reclaim the space. 211 entry_cleanup_complete(); 212 } else { 213 // We chose not to evacuate because we found sufficient immediate garbage. 214 // However, there may still be regions to promote in place, so do that now. 215 if (has_in_place_promotions(heap)) { 216 entry_promote_in_place(); 217 218 // If the promote-in-place operation was cancelled, we can have the degenerated 219 // cycle complete the operation. It will see that no evacuations are in progress, 220 // and that there are regions wanting promotion. The risk with not handling the 221 // cancellation would be failing to restore top for these regions and leaving 222 // them unable to serve allocations for the old generation. 223 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { 224 return false; 225 } 226 } 227 228 // At this point, the cycle is effectively complete. If the cycle has been cancelled here, 229 // the control thread will detect it on its next iteration and run a degenerated young cycle. 230 vmop_entry_final_roots(); 231 _abbreviated = true; 232 } 233 234 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an 235 // abbreviated cycle. 236 if (heap->mode()->is_generational()) { 237 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle(); 238 } 239 return true; 240 } 241 242 void ShenandoahConcurrentGC::vmop_entry_init_mark() { 243 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 244 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 245 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); 246 247 heap->try_inject_alloc_failure(); 248 VM_ShenandoahInitMark op(this); 249 VMThread::execute(&op); // jump to entry_init_mark() under safepoint 250 } 251 252 void ShenandoahConcurrentGC::vmop_entry_final_mark() { 253 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 254 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 255 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross); 256 257 heap->try_inject_alloc_failure(); 258 VM_ShenandoahFinalMarkStartEvac op(this); 259 VMThread::execute(&op); // jump to entry_final_mark under safepoint 260 } 261 262 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() { 263 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 264 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 265 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross); 266 267 heap->try_inject_alloc_failure(); 268 VM_ShenandoahInitUpdateRefs op(this); 269 VMThread::execute(&op); 270 } 271 272 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { 273 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 274 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 275 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross); 276 277 heap->try_inject_alloc_failure(); 278 VM_ShenandoahFinalUpdateRefs op(this); 279 VMThread::execute(&op); 280 } 281 282 void ShenandoahConcurrentGC::vmop_entry_final_roots() { 283 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 284 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 285 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); 286 287 // This phase does not use workers, no need for setup 288 heap->try_inject_alloc_failure(); 289 VM_ShenandoahFinalRoots op(this); 290 VMThread::execute(&op); 291 } 292 293 void ShenandoahConcurrentGC::entry_init_mark() { 294 const char* msg = init_mark_event_message(); 295 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); 296 EventMark em("%s", msg); 297 298 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 299 ShenandoahWorkerPolicy::calc_workers_for_init_marking(), 300 "init marking"); 301 302 op_init_mark(); 303 } 304 305 void ShenandoahConcurrentGC::entry_final_mark() { 306 const char* msg = final_mark_event_message(); 307 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); 308 EventMark em("%s", msg); 309 310 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 311 ShenandoahWorkerPolicy::calc_workers_for_final_marking(), 312 "final marking"); 313 314 op_final_mark(); 315 } 316 317 void ShenandoahConcurrentGC::entry_init_updaterefs() { 318 static const char* msg = "Pause Init Update Refs"; 319 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs); 320 EventMark em("%s", msg); 321 322 // No workers used in this phase, no setup required 323 op_init_updaterefs(); 324 } 325 326 void ShenandoahConcurrentGC::entry_final_updaterefs() { 327 static const char* msg = "Pause Final Update Refs"; 328 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs); 329 EventMark em("%s", msg); 330 331 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 332 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(), 333 "final reference update"); 334 335 op_final_updaterefs(); 336 } 337 338 void ShenandoahConcurrentGC::entry_final_roots() { 339 const char* msg = final_roots_event_message(); 340 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots); 341 EventMark em("%s", msg); 342 343 op_final_roots(); 344 } 345 346 void ShenandoahConcurrentGC::entry_reset() { 347 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 348 heap->try_inject_alloc_failure(); 349 350 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 351 { 352 const char* msg = conc_reset_event_message(); 353 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset); 354 EventMark em("%s", msg); 355 356 ShenandoahWorkerScope scope(heap->workers(), 357 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 358 msg); 359 op_reset(); 360 } 361 362 if (_do_old_gc_bootstrap) { 363 static const char* msg = "Concurrent reset (OLD)"; 364 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old); 365 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 366 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 367 msg); 368 EventMark em("%s", msg); 369 370 heap->old_generation()->prepare_gc(); 371 } 372 } 373 374 void ShenandoahConcurrentGC::entry_scan_remembered_set() { 375 if (_generation->is_young()) { 376 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 377 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 378 const char* msg = "Concurrent remembered set scanning"; 379 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset); 380 EventMark em("%s", msg); 381 382 ShenandoahWorkerScope scope(heap->workers(), 383 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(), 384 msg); 385 386 heap->try_inject_alloc_failure(); 387 _generation->scan_remembered_set(true /* is_concurrent */); 388 } 389 } 390 391 void ShenandoahConcurrentGC::entry_mark_roots() { 392 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 393 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 394 const char* msg = "Concurrent marking roots"; 395 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots); 396 EventMark em("%s", msg); 397 398 ShenandoahWorkerScope scope(heap->workers(), 399 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 400 "concurrent marking roots"); 401 402 heap->try_inject_alloc_failure(); 403 op_mark_roots(); 404 } 405 406 void ShenandoahConcurrentGC::entry_mark() { 407 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 408 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 409 const char* msg = conc_mark_event_message(); 410 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark); 411 EventMark em("%s", msg); 412 413 ShenandoahWorkerScope scope(heap->workers(), 414 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 415 "concurrent marking"); 416 417 heap->try_inject_alloc_failure(); 418 op_mark(); 419 } 420 421 void ShenandoahConcurrentGC::entry_thread_roots() { 422 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 423 static const char* msg = "Concurrent thread roots"; 424 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots); 425 EventMark em("%s", msg); 426 427 ShenandoahWorkerScope scope(heap->workers(), 428 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 429 msg); 430 431 heap->try_inject_alloc_failure(); 432 op_thread_roots(); 433 } 434 435 void ShenandoahConcurrentGC::entry_weak_refs() { 436 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 437 const char* msg = conc_weak_refs_event_message(); 438 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs); 439 EventMark em("%s", msg); 440 441 ShenandoahWorkerScope scope(heap->workers(), 442 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(), 443 "concurrent weak references"); 444 445 heap->try_inject_alloc_failure(); 446 op_weak_refs(); 447 } 448 449 void ShenandoahConcurrentGC::entry_weak_roots() { 450 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 451 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 452 const char* msg = conc_weak_roots_event_message(); 453 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots); 454 EventMark em("%s", msg); 455 456 ShenandoahWorkerScope scope(heap->workers(), 457 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 458 "concurrent weak root"); 459 460 heap->try_inject_alloc_failure(); 461 op_weak_roots(); 462 } 463 464 void ShenandoahConcurrentGC::entry_class_unloading() { 465 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 466 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 467 static const char* msg = "Concurrent class unloading"; 468 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload); 469 EventMark em("%s", msg); 470 471 ShenandoahWorkerScope scope(heap->workers(), 472 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 473 "concurrent class unloading"); 474 475 heap->try_inject_alloc_failure(); 476 op_class_unloading(); 477 } 478 479 void ShenandoahConcurrentGC::entry_strong_roots() { 480 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 481 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 482 static const char* msg = "Concurrent strong roots"; 483 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots); 484 EventMark em("%s", msg); 485 486 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots); 487 488 ShenandoahWorkerScope scope(heap->workers(), 489 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 490 "concurrent strong root"); 491 492 heap->try_inject_alloc_failure(); 493 op_strong_roots(); 494 } 495 496 void ShenandoahConcurrentGC::entry_cleanup_early() { 497 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 498 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 499 const char* msg = conc_cleanup_event_message(); 500 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */); 501 EventMark em("%s", msg); 502 503 // This phase does not use workers, no need for setup 504 heap->try_inject_alloc_failure(); 505 op_cleanup_early(); 506 } 507 508 void ShenandoahConcurrentGC::entry_evacuate() { 509 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 510 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 511 512 static const char* msg = "Concurrent evacuation"; 513 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac); 514 EventMark em("%s", msg); 515 516 ShenandoahWorkerScope scope(heap->workers(), 517 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), 518 "concurrent evacuation"); 519 520 heap->try_inject_alloc_failure(); 521 op_evacuate(); 522 } 523 524 void ShenandoahConcurrentGC::entry_promote_in_place() { 525 shenandoah_assert_generational(); 526 527 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 528 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 529 530 static const char* msg = "Promote in place"; 531 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::promote_in_place); 532 EventMark em("%s", msg); 533 534 ShenandoahWorkerScope scope(heap->workers(), 535 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), 536 "promote in place"); 537 538 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(true); 539 } 540 541 void ShenandoahConcurrentGC::entry_update_thread_roots() { 542 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 543 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 544 545 static const char* msg = "Concurrent update thread roots"; 546 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots); 547 EventMark em("%s", msg); 548 549 // No workers used in this phase, no setup required 550 heap->try_inject_alloc_failure(); 551 op_update_thread_roots(); 552 } 553 554 void ShenandoahConcurrentGC::entry_updaterefs() { 555 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 556 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 557 static const char* msg = "Concurrent update references"; 558 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs); 559 EventMark em("%s", msg); 560 561 ShenandoahWorkerScope scope(heap->workers(), 562 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(), 563 "concurrent reference update"); 564 565 heap->try_inject_alloc_failure(); 566 op_updaterefs(); 567 } 568 569 void ShenandoahConcurrentGC::entry_cleanup_complete() { 570 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 571 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 572 const char* msg = conc_cleanup_event_message(); 573 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */); 574 EventMark em("%s", msg); 575 576 // This phase does not use workers, no need for setup 577 heap->try_inject_alloc_failure(); 578 op_cleanup_complete(); 579 } 580 581 void ShenandoahConcurrentGC::op_reset() { 582 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 583 if (ShenandoahPacing) { 584 heap->pacer()->setup_for_reset(); 585 } 586 _generation->prepare_gc(); 587 } 588 589 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { 590 private: 591 ShenandoahMarkingContext* const _ctx; 592 public: 593 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} 594 595 void heap_region_do(ShenandoahHeapRegion* r) { 596 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); 597 if (r->is_active()) { 598 // Check if region needs updating its TAMS. We have updated it already during concurrent 599 // reset, so it is very likely we don't need to do another write here. Since most regions 600 // are not "active", this path is relatively rare. 601 if (_ctx->top_at_mark_start(r) != r->top()) { 602 _ctx->capture_top_at_mark_start(r); 603 } 604 } else { 605 assert(_ctx->top_at_mark_start(r) == r->top(), 606 "Region " SIZE_FORMAT " should already have correct TAMS", r->index()); 607 } 608 } 609 610 bool is_thread_safe() { return true; } 611 }; 612 613 void ShenandoahConcurrentGC::start_mark() { 614 _mark.start_mark(); 615 } 616 617 void ShenandoahConcurrentGC::op_init_mark() { 618 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 619 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 620 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); 621 622 assert(_generation->is_bitmap_clear(), "need clear marking bitmap"); 623 assert(!_generation->is_mark_complete(), "should not be complete"); 624 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 625 626 627 if (heap->mode()->is_generational()) { 628 if (_generation->is_young()) { 629 // The current implementation of swap_remembered_set() copies the write-card-table to the read-card-table. 630 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); 631 _generation->swap_remembered_set(); 632 } 633 634 if (_generation->is_global()) { 635 heap->old_generation()->cancel_gc(); 636 } else if (heap->is_concurrent_old_mark_in_progress()) { 637 // Purge the SATB buffers, transferring any valid, old pointers to the 638 // old generation mark queue. Any pointers in a young region will be 639 // abandoned. 640 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb); 641 heap->old_generation()->transfer_pointers_from_satb(); 642 } 643 } 644 645 if (ShenandoahVerify) { 646 heap->verifier()->verify_before_concmark(); 647 } 648 649 if (VerifyBeforeGC) { 650 Universe::verify(); 651 } 652 653 _generation->set_concurrent_mark_in_progress(true); 654 655 start_mark(); 656 657 if (_do_old_gc_bootstrap) { 658 shenandoah_assert_generational(); 659 // Update region state for both young and old regions 660 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 661 ShenandoahInitMarkUpdateRegionStateClosure cl; 662 heap->parallel_heap_region_iterate(&cl); 663 heap->old_generation()->ref_processor()->reset_thread_locals(); 664 } else { 665 // Update region state for only young regions 666 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 667 ShenandoahInitMarkUpdateRegionStateClosure cl; 668 _generation->parallel_heap_region_iterate(&cl); 669 } 670 671 // Weak reference processing 672 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 673 rp->reset_thread_locals(); 674 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs()); 675 676 // Make above changes visible to worker threads 677 OrderAccess::fence(); 678 679 // Arm nmethods for concurrent mark 680 ShenandoahCodeRoots::arm_nmethods_for_mark(); 681 682 ShenandoahStackWatermark::change_epoch_id(); 683 if (ShenandoahPacing) { 684 heap->pacer()->setup_for_mark(); 685 } 686 } 687 688 void ShenandoahConcurrentGC::op_mark_roots() { 689 _mark.mark_concurrent_roots(); 690 } 691 692 void ShenandoahConcurrentGC::op_mark() { 693 _mark.concurrent_mark(); 694 } 695 696 void ShenandoahConcurrentGC::op_final_mark() { 697 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 698 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 699 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 700 701 if (ShenandoahVerify) { 702 heap->verifier()->verify_roots_no_forwarded(); 703 } 704 705 if (!heap->cancelled_gc()) { 706 _mark.finish_mark(); 707 assert(!heap->cancelled_gc(), "STW mark cannot OOM"); 708 709 // Notify JVMTI that the tagmap table will need cleaning. 710 JvmtiTagMap::set_needs_cleaning(); 711 712 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been 713 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in 714 // ShenandoahGeneration and ShenandoahOldGeneration for more detail. 715 _generation->prepare_regions_and_collection_set(true /*concurrent*/); 716 717 // Has to be done after cset selection 718 heap->prepare_concurrent_roots(); 719 720 if (!heap->collection_set()->is_empty()) { 721 LogTarget(Debug, gc, cset) lt; 722 if (lt.is_enabled()) { 723 ResourceMark rm; 724 LogStream ls(lt); 725 heap->collection_set()->print_on(&ls); 726 } 727 728 if (ShenandoahVerify) { 729 heap->verifier()->verify_before_evacuation(); 730 } 731 732 heap->set_evacuation_in_progress(true); 733 // From here on, we need to update references. 734 heap->set_has_forwarded_objects(true); 735 736 // Arm nmethods/stack for concurrent processing 737 ShenandoahCodeRoots::arm_nmethods_for_evac(); 738 ShenandoahStackWatermark::change_epoch_id(); 739 740 if (ShenandoahPacing) { 741 heap->pacer()->setup_for_evac(); 742 } 743 } else { 744 if (ShenandoahVerify) { 745 if (has_in_place_promotions(heap)) { 746 heap->verifier()->verify_after_concmark_with_promotions(); 747 } else { 748 heap->verifier()->verify_after_concmark(); 749 } 750 } 751 752 if (VerifyAfterGC) { 753 Universe::verify(); 754 } 755 } 756 } 757 } 758 759 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) { 760 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions(); 761 } 762 763 template<bool GENERATIONAL> 764 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure { 765 private: 766 OopClosure* const _oops; 767 public: 768 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {} 769 770 void do_thread(Thread* thread) override { 771 JavaThread* const jt = JavaThread::cast(thread); 772 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc); 773 if (GENERATIONAL) { 774 ShenandoahThreadLocalData::enable_plab_promotions(thread); 775 } 776 } 777 }; 778 779 template<bool GENERATIONAL> 780 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask { 781 private: 782 ShenandoahJavaThreadsIterator _java_threads; 783 784 public: 785 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) : 786 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"), 787 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) { 788 } 789 790 void work(uint worker_id) override { 791 if (GENERATIONAL) { 792 Thread* worker_thread = Thread::current(); 793 ShenandoahThreadLocalData::enable_plab_promotions(worker_thread); 794 } 795 796 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure. 797 // Otherwise, may deadlock with watermark lock 798 ShenandoahContextEvacuateUpdateRootsClosure oops_cl; 799 ShenandoahConcurrentEvacThreadClosure<GENERATIONAL> thr_cl(&oops_cl); 800 _java_threads.threads_do(&thr_cl, worker_id); 801 } 802 }; 803 804 void ShenandoahConcurrentGC::op_thread_roots() { 805 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 806 assert(heap->is_evacuation_in_progress(), "Checked by caller"); 807 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots); 808 if (heap->mode()->is_generational()) { 809 ShenandoahConcurrentEvacUpdateThreadTask<true> task(heap->workers()->active_workers()); 810 heap->workers()->run_task(&task); 811 } else { 812 ShenandoahConcurrentEvacUpdateThreadTask<false> task(heap->workers()->active_workers()); 813 heap->workers()->run_task(&task); 814 } 815 } 816 817 void ShenandoahConcurrentGC::op_weak_refs() { 818 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 819 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 820 // Concurrent weak refs processing 821 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs); 822 if (heap->gc_cause() == GCCause::_wb_breakpoint) { 823 ShenandoahBreakpoint::at_after_reference_processing_started(); 824 } 825 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */); 826 } 827 828 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure { 829 private: 830 ShenandoahHeap* const _heap; 831 ShenandoahMarkingContext* const _mark_context; 832 bool _evac_in_progress; 833 Thread* const _thread; 834 835 public: 836 ShenandoahEvacUpdateCleanupOopStorageRootsClosure(); 837 void do_oop(oop* p); 838 void do_oop(narrowOop* p); 839 }; 840 841 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() : 842 _heap(ShenandoahHeap::heap()), 843 _mark_context(ShenandoahHeap::heap()->marking_context()), 844 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()), 845 _thread(Thread::current()) { 846 } 847 848 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { 849 const oop obj = RawAccess<>::oop_load(p); 850 if (!CompressedOops::is_null(obj)) { 851 if (!_mark_context->is_marked(obj)) { 852 shenandoah_assert_generations_reconciled(); 853 if (_heap->is_in_active_generation(obj)) { 854 // Note: The obj is dead here. Do not touch it, just clear. 855 ShenandoahHeap::atomic_clear_oop(p, obj); 856 } 857 } else if (_evac_in_progress && _heap->in_collection_set(obj)) { 858 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 859 if (resolved == obj) { 860 resolved = _heap->evacuate_object(obj, _thread); 861 } 862 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc()); 863 ShenandoahHeap::atomic_update_oop(resolved, p, obj); 864 } 865 } 866 } 867 868 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) { 869 ShouldNotReachHere(); 870 } 871 872 class ShenandoahIsCLDAliveClosure : public CLDClosure { 873 public: 874 void do_cld(ClassLoaderData* cld) { 875 cld->is_alive(); 876 } 877 }; 878 879 class ShenandoahIsNMethodAliveClosure: public NMethodClosure { 880 public: 881 void do_nmethod(nmethod* n) { 882 n->is_unloading(); 883 } 884 }; 885 886 // This task not only evacuates/updates marked weak roots, but also "null" 887 // dead weak roots. 888 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask { 889 private: 890 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots; 891 892 // Roots related to concurrent class unloading 893 ShenandoahClassLoaderDataRoots<true /* concurrent */> 894 _cld_roots; 895 ShenandoahConcurrentNMethodIterator _nmethod_itr; 896 ShenandoahPhaseTimings::Phase _phase; 897 898 public: 899 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 900 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"), 901 _vm_roots(phase), 902 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 903 _nmethod_itr(ShenandoahCodeRoots::table()), 904 _phase(phase) {} 905 906 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() { 907 // Notify runtime data structures of potentially dead oops 908 _vm_roots.report_num_dead(); 909 } 910 911 void work(uint worker_id) { 912 ShenandoahConcurrentWorkerSession worker_session(worker_id); 913 ShenandoahSuspendibleThreadSetJoiner sts_join; 914 { 915 ShenandoahEvacOOMScope oom; 916 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration 917 // may race against OopStorage::release() calls. 918 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl; 919 _vm_roots.oops_do(&cl, worker_id); 920 } 921 922 // If we are going to perform concurrent class unloading later on, we need to 923 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we 924 // can cleanup immediate garbage sooner. 925 if (ShenandoahHeap::heap()->unload_classes()) { 926 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the 927 // CLD's holder or evacuate it. 928 { 929 ShenandoahIsCLDAliveClosure is_cld_alive; 930 _cld_roots.cld_do(&is_cld_alive, worker_id); 931 } 932 933 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods. 934 // The closure calls nmethod->is_unloading(). The is_unloading 935 // state is cached, therefore, during concurrent class unloading phase, 936 // we will not touch the metadata of unloading nmethods 937 { 938 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 939 ShenandoahIsNMethodAliveClosure is_nmethod_alive; 940 _nmethod_itr.nmethods_do(&is_nmethod_alive); 941 } 942 } 943 } 944 }; 945 946 void ShenandoahConcurrentGC::op_weak_roots() { 947 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 948 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 949 // Concurrent weak root processing 950 { 951 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work); 952 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work); 953 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work); 954 heap->workers()->run_task(&task); 955 } 956 957 // Perform handshake to flush out dead oops 958 { 959 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous); 960 heap->rendezvous_threads("Shenandoah Concurrent Weak Roots"); 961 } 962 // We can only toggle concurrent_weak_root_in_progress flag 963 // at a safepoint, so that mutators see a consistent 964 // value. The flag will be cleared at the next safepoint. 965 } 966 967 void ShenandoahConcurrentGC::op_class_unloading() { 968 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 969 assert (heap->is_concurrent_weak_root_in_progress() && 970 heap->unload_classes(), 971 "Checked by caller"); 972 heap->do_class_unloading(); 973 } 974 975 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure { 976 private: 977 BarrierSetNMethod* const _bs; 978 ShenandoahEvacuateUpdateMetadataClosure _cl; 979 980 public: 981 ShenandoahEvacUpdateCodeCacheClosure() : 982 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()), 983 _cl() { 984 } 985 986 void do_nmethod(nmethod* n) { 987 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n); 988 ShenandoahReentrantLocker locker(data->lock()); 989 // Setup EvacOOM scope below reentrant lock to avoid deadlock with 990 // nmethod_entry_barrier 991 ShenandoahEvacOOMScope oom; 992 data->oops_do(&_cl, true/*fix relocation*/); 993 _bs->disarm(n); 994 } 995 }; 996 997 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask { 998 private: 999 ShenandoahPhaseTimings::Phase _phase; 1000 ShenandoahVMRoots<true /*concurrent*/> _vm_roots; 1001 ShenandoahClassLoaderDataRoots<true /*concurrent*/> 1002 _cld_roots; 1003 ShenandoahConcurrentNMethodIterator _nmethod_itr; 1004 1005 public: 1006 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 1007 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"), 1008 _phase(phase), 1009 _vm_roots(phase), 1010 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 1011 _nmethod_itr(ShenandoahCodeRoots::table()) {} 1012 1013 void work(uint worker_id) { 1014 ShenandoahConcurrentWorkerSession worker_session(worker_id); 1015 { 1016 ShenandoahEvacOOMScope oom; 1017 { 1018 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration 1019 // may race against OopStorage::release() calls. 1020 ShenandoahContextEvacuateUpdateRootsClosure cl; 1021 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id); 1022 } 1023 1024 { 1025 ShenandoahEvacuateUpdateMetadataClosure cl; 1026 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong); 1027 _cld_roots.cld_do(&clds, worker_id); 1028 } 1029 } 1030 1031 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier. 1032 if (!ShenandoahHeap::heap()->unload_classes()) { 1033 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 1034 ShenandoahEvacUpdateCodeCacheClosure cl; 1035 _nmethod_itr.nmethods_do(&cl); 1036 } 1037 } 1038 }; 1039 1040 void ShenandoahConcurrentGC::op_strong_roots() { 1041 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1042 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller"); 1043 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots); 1044 heap->workers()->run_task(&task); 1045 heap->set_concurrent_strong_root_in_progress(false); 1046 } 1047 1048 void ShenandoahConcurrentGC::op_cleanup_early() { 1049 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1050 } 1051 1052 void ShenandoahConcurrentGC::op_evacuate() { 1053 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/); 1054 } 1055 1056 void ShenandoahConcurrentGC::op_init_updaterefs() { 1057 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1058 heap->set_evacuation_in_progress(false); 1059 heap->set_concurrent_weak_root_in_progress(false); 1060 heap->prepare_update_heap_references(true /*concurrent*/); 1061 heap->set_update_refs_in_progress(true); 1062 if (ShenandoahVerify) { 1063 heap->verifier()->verify_before_updaterefs(); 1064 } 1065 if (ShenandoahPacing) { 1066 heap->pacer()->setup_for_updaterefs(); 1067 } 1068 } 1069 1070 void ShenandoahConcurrentGC::op_updaterefs() { 1071 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/); 1072 } 1073 1074 class ShenandoahUpdateThreadClosure : public HandshakeClosure { 1075 private: 1076 ShenandoahUpdateRefsClosure _cl; 1077 public: 1078 ShenandoahUpdateThreadClosure(); 1079 void do_thread(Thread* thread); 1080 }; 1081 1082 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() : 1083 HandshakeClosure("Shenandoah Update Thread Roots") { 1084 } 1085 1086 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) { 1087 if (thread->is_Java_thread()) { 1088 JavaThread* jt = JavaThread::cast(thread); 1089 ResourceMark rm; 1090 jt->oops_do(&_cl, nullptr); 1091 } 1092 } 1093 1094 void ShenandoahConcurrentGC::op_update_thread_roots() { 1095 ShenandoahUpdateThreadClosure cl; 1096 Handshake::execute(&cl); 1097 } 1098 1099 void ShenandoahConcurrentGC::op_final_updaterefs() { 1100 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1101 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); 1102 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references"); 1103 1104 heap->finish_concurrent_roots(); 1105 1106 // Clear cancelled GC, if set. On cancellation path, the block before would handle 1107 // everything. 1108 if (heap->cancelled_gc()) { 1109 heap->clear_cancelled_gc(true /* clear oom handler */); 1110 } 1111 1112 // Has to be done before cset is clear 1113 if (ShenandoahVerify) { 1114 heap->verifier()->verify_roots_in_to_space(); 1115 } 1116 1117 // If we are running in generational mode and this is an aging cycle, this will also age active 1118 // regions that haven't been used for allocation. 1119 heap->update_heap_region_states(true /*concurrent*/); 1120 1121 heap->set_update_refs_in_progress(false); 1122 heap->set_has_forwarded_objects(false); 1123 1124 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) { 1125 // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to 1126 // objects in the collection set. After those objects are evacuated, the pointers in the 1127 // SATB are no longer safe. Once we have finished update references, we are guaranteed that 1128 // no more writes to the collection set are possible. 1129 // 1130 // This will transfer any old pointers in _active_ regions from the SATB to the old gen 1131 // mark queues. All other pointers will be discarded. This would also discard any pointers 1132 // in old regions that were included in a mixed evacuation. We aren't using the SATB filter 1133 // methods here because we cannot control when they execute. If the SATB filter runs _after_ 1134 // a region has been recycled, we will not be able to detect the bad pointer. 1135 // 1136 // We are not concerned about skipping this step in abbreviated cycles because regions 1137 // with no live objects cannot have been written to and so cannot have entries in the SATB 1138 // buffers. 1139 heap->old_generation()->transfer_pointers_from_satb(); 1140 1141 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for 1142 // entire regions. Both of these relevant operations occur before final update refs. 1143 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false); 1144 } 1145 1146 if (ShenandoahVerify) { 1147 heap->verifier()->verify_after_updaterefs(); 1148 } 1149 1150 if (VerifyAfterGC) { 1151 Universe::verify(); 1152 } 1153 1154 heap->rebuild_free_set(true /*concurrent*/); 1155 } 1156 1157 void ShenandoahConcurrentGC::op_final_roots() { 1158 1159 ShenandoahHeap *heap = ShenandoahHeap::heap(); 1160 heap->set_concurrent_weak_root_in_progress(false); 1161 heap->set_evacuation_in_progress(false); 1162 1163 if (heap->mode()->is_generational()) { 1164 // If the cycle was shortened for having enough immediate garbage, this could be 1165 // the last GC safepoint before concurrent marking of old resumes. We must be sure 1166 // that old mark threads don't see any pointers to garbage in the SATB buffers. 1167 if (heap->is_concurrent_old_mark_in_progress()) { 1168 heap->old_generation()->transfer_pointers_from_satb(); 1169 } 1170 1171 if (!_generation->is_old()) { 1172 ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context()); 1173 } 1174 } 1175 } 1176 1177 void ShenandoahConcurrentGC::op_cleanup_complete() { 1178 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1179 } 1180 1181 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) { 1182 if (ShenandoahHeap::heap()->cancelled_gc()) { 1183 _degen_point = point; 1184 return true; 1185 } 1186 return false; 1187 } 1188 1189 const char* ShenandoahConcurrentGC::init_mark_event_message() const { 1190 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1191 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); 1192 if (heap->unload_classes()) { 1193 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)"); 1194 } else { 1195 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", ""); 1196 } 1197 } 1198 1199 const char* ShenandoahConcurrentGC::final_mark_event_message() const { 1200 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1201 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1202 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running"); 1203 1204 if (heap->unload_classes()) { 1205 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)"); 1206 } else { 1207 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", ""); 1208 } 1209 } 1210 1211 const char* ShenandoahConcurrentGC::conc_mark_event_message() const { 1212 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1213 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1214 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running"); 1215 if (heap->unload_classes()) { 1216 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)"); 1217 } else { 1218 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", ""); 1219 } 1220 } 1221 1222 const char* ShenandoahConcurrentGC::conc_reset_event_message() const { 1223 if (ShenandoahHeap::heap()->unload_classes()) { 1224 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)"); 1225 } else { 1226 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", ""); 1227 } 1228 } 1229 1230 const char* ShenandoahConcurrentGC::final_roots_event_message() const { 1231 if (ShenandoahHeap::heap()->unload_classes()) { 1232 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)"); 1233 } else { 1234 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", ""); 1235 } 1236 } 1237 1238 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const { 1239 if (ShenandoahHeap::heap()->unload_classes()) { 1240 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)"); 1241 } else { 1242 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", ""); 1243 } 1244 } 1245 1246 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const { 1247 if (ShenandoahHeap::heap()->unload_classes()) { 1248 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)"); 1249 } else { 1250 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", ""); 1251 } 1252 } 1253 1254 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const { 1255 if (ShenandoahHeap::heap()->unload_classes()) { 1256 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)"); 1257 } else { 1258 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", ""); 1259 } 1260 }