1 /* 2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 29 #include "gc/shared/barrierSetNMethod.hpp" 30 #include "gc/shared/collectorCounters.hpp" 31 #include "gc/shared/continuationGCSupport.inline.hpp" 32 #include "gc/shenandoah/shenandoahBreakpoint.hpp" 33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp" 35 #include "gc/shenandoah/shenandoahFreeSet.hpp" 36 #include "gc/shenandoah/shenandoahGeneration.hpp" 37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 38 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 40 #include "gc/shenandoah/shenandoahLock.hpp" 41 #include "gc/shenandoah/shenandoahMark.inline.hpp" 42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 45 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 46 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 47 #include "gc/shenandoah/shenandoahStackWatermark.hpp" 48 #include "gc/shenandoah/shenandoahUtils.hpp" 49 #include "gc/shenandoah/shenandoahVerifier.hpp" 50 #include "gc/shenandoah/shenandoahVMOperations.hpp" 51 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 52 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 53 #include "memory/allocation.hpp" 54 #include "prims/jvmtiTagMap.hpp" 55 #include "runtime/vmThread.hpp" 56 #include "utilities/events.hpp" 57 58 // Breakpoint support 59 class ShenandoahBreakpointGCScope : public StackObj { 60 private: 61 const GCCause::Cause _cause; 62 public: 63 ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) { 64 if (cause == GCCause::_wb_breakpoint) { 65 ShenandoahBreakpoint::start_gc(); 66 ShenandoahBreakpoint::at_before_gc(); 67 } 68 } 69 70 ~ShenandoahBreakpointGCScope() { 71 if (_cause == GCCause::_wb_breakpoint) { 72 ShenandoahBreakpoint::at_after_gc(); 73 } 74 } 75 }; 76 77 class ShenandoahBreakpointMarkScope : public StackObj { 78 private: 79 const GCCause::Cause _cause; 80 public: 81 ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) { 82 if (_cause == GCCause::_wb_breakpoint) { 83 ShenandoahBreakpoint::at_after_marking_started(); 84 } 85 } 86 87 ~ShenandoahBreakpointMarkScope() { 88 if (_cause == GCCause::_wb_breakpoint) { 89 ShenandoahBreakpoint::at_before_marking_completed(); 90 } 91 } 92 }; 93 94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : 95 _mark(generation), 96 _degen_point(ShenandoahDegenPoint::_degenerated_unset), 97 _abbreviated(false), 98 _do_old_gc_bootstrap(do_old_gc_bootstrap), 99 _generation(generation) { 100 } 101 102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { 103 return _degen_point; 104 } 105 106 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { 107 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 108 109 ShenandoahBreakpointGCScope breakpoint_gc_scope(cause); 110 111 // Reset for upcoming marking 112 entry_reset(); 113 114 // Start initial mark under STW 115 vmop_entry_init_mark(); 116 117 { 118 ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause); 119 120 // Reset task queue stats here, rather than in mark_concurrent_roots, 121 // because remembered set scan will `push` oops into the queues and 122 // resetting after this happens will lose those counts. 123 TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); 124 125 // Concurrent remembered set scanning 126 entry_scan_remembered_set(); 127 128 // Concurrent mark roots 129 entry_mark_roots(); 130 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) { 131 return false; 132 } 133 134 // Continue concurrent mark 135 entry_mark(); 136 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { 137 return false; 138 } 139 } 140 141 // Complete marking under STW, and start evacuation 142 vmop_entry_final_mark(); 143 144 // If the GC was cancelled just before final mark (but after the preceding cancellation check), 145 // then the safepoint operation will do nothing and the concurrent mark will still be in progress. 146 // In this case it is safe (and necessary) to resume the degenerated cycle from the marking phase. 147 // 148 // On the other hand, if the GC is cancelled after final mark (but before this check), then the 149 // final mark safepoint operation will have finished the mark (setting concurrent mark in progress 150 // to false). In this case (final mark has completed), we need control to fall past the next 151 // cancellation check and resume the degenerated cycle from the evacuation phase. 152 if (_generation->is_concurrent_mark_in_progress()) { 153 // If the concurrent mark is still in progress after the final mark safepoint, then the GC has 154 // been cancelled. The degenerated cycle must resume from the marking phase. Without this check, 155 // the non-generational mode may fall all the way to the end of this collect routine without 156 // having done anything (besides mark most of the heap). Without having collected anything, we 157 // can expect an 'out of cycle' degenerated GC which will again mark the entire heap. This is 158 // not optimal. 159 // For the generational mode, we cannot allow this. The generational mode relies on marking 160 // (including the final mark) to rebuild portions of the card table. If the generational mode does 161 // not complete marking after it has swapped the card tables, the root set on subsequent GCs will 162 // be incomplete, heap corruption may follow. 163 bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark); 164 assert(cancelled, "GC must have been cancelled between concurrent and final mark"); 165 return false; 166 } 167 168 // Concurrent stack processing 169 if (heap->is_evacuation_in_progress()) { 170 entry_thread_roots(); 171 } 172 173 // Process weak roots that might still point to regions that would be broken by cleanup 174 if (heap->is_concurrent_weak_root_in_progress()) { 175 entry_weak_refs(); 176 entry_weak_roots(); 177 } 178 179 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim 180 // the space. This would be the last action if there is nothing to evacuate. Note that 181 // we will not age young-gen objects in the case that we skip evacuation. 182 entry_cleanup_early(); 183 184 heap->free_set()->log_status_under_lock(); 185 186 // Perform concurrent class unloading 187 if (heap->unload_classes() && 188 heap->is_concurrent_weak_root_in_progress()) { 189 entry_class_unloading(); 190 } 191 192 // Processing strong roots 193 // This may be skipped if there is nothing to update/evacuate. 194 // If so, strong_root_in_progress would be unset. 195 if (heap->is_concurrent_strong_root_in_progress()) { 196 entry_strong_roots(); 197 } 198 199 // Continue the cycle with evacuation and optional update-refs. 200 // This may be skipped if there is nothing to evacuate. 201 // If so, evac_in_progress would be unset by collection set preparation code. 202 if (heap->is_evacuation_in_progress()) { 203 // Concurrently evacuate 204 entry_evacuate(); 205 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { 206 return false; 207 } 208 209 // Perform update-refs phase. 210 vmop_entry_init_updaterefs(); 211 entry_updaterefs(); 212 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) { 213 return false; 214 } 215 216 // Concurrent update thread roots 217 entry_update_thread_roots(); 218 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) { 219 return false; 220 } 221 222 vmop_entry_final_updaterefs(); 223 224 // Update references freed up collection set, kick the cleanup to reclaim the space. 225 entry_cleanup_complete(); 226 } else { 227 // We chose not to evacuate because we found sufficient immediate garbage. 228 // However, there may still be regions to promote in place, so do that now. 229 if (has_in_place_promotions(heap)) { 230 entry_promote_in_place(); 231 232 // If the promote-in-place operation was cancelled, we can have the degenerated 233 // cycle complete the operation. It will see that no evacuations are in progress, 234 // and that there are regions wanting promotion. The risk with not handling the 235 // cancellation would be failing to restore top for these regions and leaving 236 // them unable to serve allocations for the old generation. 237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { 238 return false; 239 } 240 } 241 242 // At this point, the cycle is effectively complete. If the cycle has been cancelled here, 243 // the control thread will detect it on its next iteration and run a degenerated young cycle. 244 vmop_entry_final_roots(); 245 _abbreviated = true; 246 } 247 248 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an 249 // abbreviated cycle. 250 if (heap->mode()->is_generational()) { 251 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle(); 252 } 253 return true; 254 } 255 256 void ShenandoahConcurrentGC::vmop_entry_init_mark() { 257 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 258 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 259 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); 260 261 heap->try_inject_alloc_failure(); 262 VM_ShenandoahInitMark op(this); 263 VMThread::execute(&op); // jump to entry_init_mark() under safepoint 264 } 265 266 void ShenandoahConcurrentGC::vmop_entry_final_mark() { 267 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 268 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 269 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross); 270 271 heap->try_inject_alloc_failure(); 272 VM_ShenandoahFinalMarkStartEvac op(this); 273 VMThread::execute(&op); // jump to entry_final_mark under safepoint 274 } 275 276 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() { 277 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 278 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 279 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross); 280 281 heap->try_inject_alloc_failure(); 282 VM_ShenandoahInitUpdateRefs op(this); 283 VMThread::execute(&op); 284 } 285 286 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { 287 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 288 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 289 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross); 290 291 heap->try_inject_alloc_failure(); 292 VM_ShenandoahFinalUpdateRefs op(this); 293 VMThread::execute(&op); 294 } 295 296 void ShenandoahConcurrentGC::vmop_entry_final_roots() { 297 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 298 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); 299 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); 300 301 // This phase does not use workers, no need for setup 302 heap->try_inject_alloc_failure(); 303 VM_ShenandoahFinalRoots op(this); 304 VMThread::execute(&op); 305 } 306 307 void ShenandoahConcurrentGC::entry_init_mark() { 308 const char* msg = init_mark_event_message(); 309 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); 310 EventMark em("%s", msg); 311 312 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 313 ShenandoahWorkerPolicy::calc_workers_for_init_marking(), 314 "init marking"); 315 316 op_init_mark(); 317 } 318 319 void ShenandoahConcurrentGC::entry_final_mark() { 320 const char* msg = final_mark_event_message(); 321 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); 322 EventMark em("%s", msg); 323 324 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 325 ShenandoahWorkerPolicy::calc_workers_for_final_marking(), 326 "final marking"); 327 328 op_final_mark(); 329 } 330 331 void ShenandoahConcurrentGC::entry_init_updaterefs() { 332 static const char* msg = "Pause Init Update Refs"; 333 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs); 334 EventMark em("%s", msg); 335 336 // No workers used in this phase, no setup required 337 op_init_updaterefs(); 338 } 339 340 void ShenandoahConcurrentGC::entry_final_updaterefs() { 341 static const char* msg = "Pause Final Update Refs"; 342 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs); 343 EventMark em("%s", msg); 344 345 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 346 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(), 347 "final reference update"); 348 349 op_final_updaterefs(); 350 } 351 352 void ShenandoahConcurrentGC::entry_final_roots() { 353 const char* msg = final_roots_event_message(); 354 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots); 355 EventMark em("%s", msg); 356 357 op_final_roots(); 358 } 359 360 void ShenandoahConcurrentGC::entry_reset() { 361 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 362 heap->try_inject_alloc_failure(); 363 364 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 365 { 366 const char* msg = conc_reset_event_message(); 367 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset); 368 EventMark em("%s", msg); 369 370 ShenandoahWorkerScope scope(heap->workers(), 371 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 372 msg); 373 op_reset(); 374 } 375 376 if (_do_old_gc_bootstrap) { 377 static const char* msg = "Concurrent reset (OLD)"; 378 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old); 379 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), 380 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), 381 msg); 382 EventMark em("%s", msg); 383 384 heap->old_generation()->prepare_gc(); 385 } 386 } 387 388 void ShenandoahConcurrentGC::entry_scan_remembered_set() { 389 if (_generation->is_young()) { 390 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 391 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 392 const char* msg = "Concurrent remembered set scanning"; 393 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset); 394 EventMark em("%s", msg); 395 396 ShenandoahWorkerScope scope(heap->workers(), 397 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(), 398 msg); 399 400 heap->try_inject_alloc_failure(); 401 _generation->scan_remembered_set(true /* is_concurrent */); 402 } 403 } 404 405 void ShenandoahConcurrentGC::entry_mark_roots() { 406 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 407 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 408 const char* msg = "Concurrent marking roots"; 409 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots); 410 EventMark em("%s", msg); 411 412 ShenandoahWorkerScope scope(heap->workers(), 413 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 414 "concurrent marking roots"); 415 416 heap->try_inject_alloc_failure(); 417 op_mark_roots(); 418 } 419 420 void ShenandoahConcurrentGC::entry_mark() { 421 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 422 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 423 const char* msg = conc_mark_event_message(); 424 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark); 425 EventMark em("%s", msg); 426 427 ShenandoahWorkerScope scope(heap->workers(), 428 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), 429 "concurrent marking"); 430 431 heap->try_inject_alloc_failure(); 432 op_mark(); 433 } 434 435 void ShenandoahConcurrentGC::entry_thread_roots() { 436 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 437 static const char* msg = "Concurrent thread roots"; 438 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots); 439 EventMark em("%s", msg); 440 441 ShenandoahWorkerScope scope(heap->workers(), 442 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 443 msg); 444 445 heap->try_inject_alloc_failure(); 446 op_thread_roots(); 447 } 448 449 void ShenandoahConcurrentGC::entry_weak_refs() { 450 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 451 const char* msg = conc_weak_refs_event_message(); 452 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs); 453 EventMark em("%s", msg); 454 455 ShenandoahWorkerScope scope(heap->workers(), 456 ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(), 457 "concurrent weak references"); 458 459 heap->try_inject_alloc_failure(); 460 op_weak_refs(); 461 } 462 463 void ShenandoahConcurrentGC::entry_weak_roots() { 464 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 465 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 466 const char* msg = conc_weak_roots_event_message(); 467 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots); 468 EventMark em("%s", msg); 469 470 ShenandoahWorkerScope scope(heap->workers(), 471 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 472 "concurrent weak root"); 473 474 heap->try_inject_alloc_failure(); 475 op_weak_roots(); 476 } 477 478 void ShenandoahConcurrentGC::entry_class_unloading() { 479 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 480 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 481 static const char* msg = "Concurrent class unloading"; 482 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload); 483 EventMark em("%s", msg); 484 485 ShenandoahWorkerScope scope(heap->workers(), 486 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 487 "concurrent class unloading"); 488 489 heap->try_inject_alloc_failure(); 490 op_class_unloading(); 491 } 492 493 void ShenandoahConcurrentGC::entry_strong_roots() { 494 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 495 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 496 static const char* msg = "Concurrent strong roots"; 497 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots); 498 EventMark em("%s", msg); 499 500 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots); 501 502 ShenandoahWorkerScope scope(heap->workers(), 503 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(), 504 "concurrent strong root"); 505 506 heap->try_inject_alloc_failure(); 507 op_strong_roots(); 508 } 509 510 void ShenandoahConcurrentGC::entry_cleanup_early() { 511 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 512 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 513 const char* msg = conc_cleanup_event_message(); 514 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */); 515 EventMark em("%s", msg); 516 517 // This phase does not use workers, no need for setup 518 heap->try_inject_alloc_failure(); 519 op_cleanup_early(); 520 } 521 522 void ShenandoahConcurrentGC::entry_evacuate() { 523 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 524 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 525 526 static const char* msg = "Concurrent evacuation"; 527 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac); 528 EventMark em("%s", msg); 529 530 ShenandoahWorkerScope scope(heap->workers(), 531 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), 532 "concurrent evacuation"); 533 534 heap->try_inject_alloc_failure(); 535 op_evacuate(); 536 } 537 538 void ShenandoahConcurrentGC::entry_promote_in_place() { 539 shenandoah_assert_generational(); 540 541 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 542 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 543 544 static const char* msg = "Promote in place"; 545 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::promote_in_place); 546 EventMark em("%s", msg); 547 548 ShenandoahWorkerScope scope(heap->workers(), 549 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), 550 "promote in place"); 551 552 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(true); 553 } 554 555 void ShenandoahConcurrentGC::entry_update_thread_roots() { 556 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 557 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 558 559 static const char* msg = "Concurrent update thread roots"; 560 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots); 561 EventMark em("%s", msg); 562 563 // No workers used in this phase, no setup required 564 heap->try_inject_alloc_failure(); 565 op_update_thread_roots(); 566 } 567 568 void ShenandoahConcurrentGC::entry_updaterefs() { 569 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 570 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 571 static const char* msg = "Concurrent update references"; 572 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs); 573 EventMark em("%s", msg); 574 575 ShenandoahWorkerScope scope(heap->workers(), 576 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(), 577 "concurrent reference update"); 578 579 heap->try_inject_alloc_failure(); 580 op_updaterefs(); 581 } 582 583 void ShenandoahConcurrentGC::entry_cleanup_complete() { 584 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 585 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 586 const char* msg = conc_cleanup_event_message(); 587 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */); 588 EventMark em("%s", msg); 589 590 // This phase does not use workers, no need for setup 591 heap->try_inject_alloc_failure(); 592 op_cleanup_complete(); 593 } 594 595 void ShenandoahConcurrentGC::op_reset() { 596 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 597 if (ShenandoahPacing) { 598 heap->pacer()->setup_for_reset(); 599 } 600 _generation->prepare_gc(); 601 } 602 603 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { 604 private: 605 ShenandoahMarkingContext* const _ctx; 606 public: 607 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} 608 609 void heap_region_do(ShenandoahHeapRegion* r) { 610 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); 611 if (r->is_active()) { 612 // Check if region needs updating its TAMS. We have updated it already during concurrent 613 // reset, so it is very likely we don't need to do another write here. Since most regions 614 // are not "active", this path is relatively rare. 615 if (_ctx->top_at_mark_start(r) != r->top()) { 616 _ctx->capture_top_at_mark_start(r); 617 } 618 } else { 619 assert(_ctx->top_at_mark_start(r) == r->top(), 620 "Region " SIZE_FORMAT " should already have correct TAMS", r->index()); 621 } 622 } 623 624 bool is_thread_safe() { return true; } 625 }; 626 627 void ShenandoahConcurrentGC::start_mark() { 628 _mark.start_mark(); 629 } 630 631 void ShenandoahConcurrentGC::op_init_mark() { 632 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 633 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 634 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); 635 636 assert(_generation->is_bitmap_clear(), "need clear marking bitmap"); 637 assert(!_generation->is_mark_complete(), "should not be complete"); 638 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 639 640 641 if (heap->mode()->is_generational()) { 642 if (_generation->is_young()) { 643 // The current implementation of swap_remembered_set() copies the write-card-table to the read-card-table. 644 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); 645 _generation->swap_remembered_set(); 646 } 647 648 if (_generation->is_global()) { 649 heap->old_generation()->cancel_gc(); 650 } else if (heap->is_concurrent_old_mark_in_progress()) { 651 // Purge the SATB buffers, transferring any valid, old pointers to the 652 // old generation mark queue. Any pointers in a young region will be 653 // abandoned. 654 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb); 655 heap->old_generation()->transfer_pointers_from_satb(); 656 } 657 } 658 659 if (ShenandoahVerify) { 660 heap->verifier()->verify_before_concmark(); 661 } 662 663 if (VerifyBeforeGC) { 664 Universe::verify(); 665 } 666 667 _generation->set_concurrent_mark_in_progress(true); 668 669 start_mark(); 670 671 if (_do_old_gc_bootstrap) { 672 shenandoah_assert_generational(); 673 // Update region state for both young and old regions 674 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 675 ShenandoahInitMarkUpdateRegionStateClosure cl; 676 heap->parallel_heap_region_iterate(&cl); 677 heap->old_generation()->ref_processor()->reset_thread_locals(); 678 } else { 679 // Update region state for only young regions 680 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); 681 ShenandoahInitMarkUpdateRegionStateClosure cl; 682 _generation->parallel_heap_region_iterate(&cl); 683 } 684 685 // Weak reference processing 686 ShenandoahReferenceProcessor* rp = _generation->ref_processor(); 687 rp->reset_thread_locals(); 688 rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs()); 689 690 // Make above changes visible to worker threads 691 OrderAccess::fence(); 692 693 // Arm nmethods for concurrent mark 694 ShenandoahCodeRoots::arm_nmethods_for_mark(); 695 696 ShenandoahStackWatermark::change_epoch_id(); 697 if (ShenandoahPacing) { 698 heap->pacer()->setup_for_mark(); 699 } 700 } 701 702 void ShenandoahConcurrentGC::op_mark_roots() { 703 _mark.mark_concurrent_roots(); 704 } 705 706 void ShenandoahConcurrentGC::op_mark() { 707 _mark.concurrent_mark(); 708 } 709 710 void ShenandoahConcurrentGC::op_final_mark() { 711 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 712 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); 713 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); 714 715 if (ShenandoahVerify) { 716 heap->verifier()->verify_roots_no_forwarded(); 717 } 718 719 if (!heap->cancelled_gc()) { 720 _mark.finish_mark(); 721 assert(!heap->cancelled_gc(), "STW mark cannot OOM"); 722 723 // Notify JVMTI that the tagmap table will need cleaning. 724 JvmtiTagMap::set_needs_cleaning(); 725 726 // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been 727 // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in 728 // ShenandoahGeneration and ShenandoahOldGeneration for more detail. 729 _generation->prepare_regions_and_collection_set(true /*concurrent*/); 730 731 // Has to be done after cset selection 732 heap->prepare_concurrent_roots(); 733 734 if (!heap->collection_set()->is_empty()) { 735 LogTarget(Debug, gc, cset) lt; 736 if (lt.is_enabled()) { 737 ResourceMark rm; 738 LogStream ls(lt); 739 heap->collection_set()->print_on(&ls); 740 } 741 742 if (ShenandoahVerify) { 743 heap->verifier()->verify_before_evacuation(); 744 } 745 746 heap->set_evacuation_in_progress(true); 747 // From here on, we need to update references. 748 heap->set_has_forwarded_objects(true); 749 750 // Arm nmethods/stack for concurrent processing 751 ShenandoahCodeRoots::arm_nmethods_for_evac(); 752 ShenandoahStackWatermark::change_epoch_id(); 753 754 if (ShenandoahPacing) { 755 heap->pacer()->setup_for_evac(); 756 } 757 } else { 758 if (ShenandoahVerify) { 759 if (has_in_place_promotions(heap)) { 760 heap->verifier()->verify_after_concmark_with_promotions(); 761 } else { 762 heap->verifier()->verify_after_concmark(); 763 } 764 } 765 766 if (VerifyAfterGC) { 767 Universe::verify(); 768 } 769 } 770 } 771 } 772 773 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) { 774 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions(); 775 } 776 777 template<bool GENERATIONAL> 778 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure { 779 private: 780 OopClosure* const _oops; 781 public: 782 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {} 783 784 void do_thread(Thread* thread) override { 785 JavaThread* const jt = JavaThread::cast(thread); 786 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc); 787 if (GENERATIONAL) { 788 ShenandoahThreadLocalData::enable_plab_promotions(thread); 789 } 790 } 791 }; 792 793 template<bool GENERATIONAL> 794 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask { 795 private: 796 ShenandoahJavaThreadsIterator _java_threads; 797 798 public: 799 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) : 800 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"), 801 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) { 802 } 803 804 void work(uint worker_id) override { 805 if (GENERATIONAL) { 806 Thread* worker_thread = Thread::current(); 807 ShenandoahThreadLocalData::enable_plab_promotions(worker_thread); 808 } 809 810 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure. 811 // Otherwise, may deadlock with watermark lock 812 ShenandoahContextEvacuateUpdateRootsClosure oops_cl; 813 ShenandoahConcurrentEvacThreadClosure<GENERATIONAL> thr_cl(&oops_cl); 814 _java_threads.threads_do(&thr_cl, worker_id); 815 } 816 }; 817 818 void ShenandoahConcurrentGC::op_thread_roots() { 819 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 820 assert(heap->is_evacuation_in_progress(), "Checked by caller"); 821 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots); 822 if (heap->mode()->is_generational()) { 823 ShenandoahConcurrentEvacUpdateThreadTask<true> task(heap->workers()->active_workers()); 824 heap->workers()->run_task(&task); 825 } else { 826 ShenandoahConcurrentEvacUpdateThreadTask<false> task(heap->workers()->active_workers()); 827 heap->workers()->run_task(&task); 828 } 829 } 830 831 void ShenandoahConcurrentGC::op_weak_refs() { 832 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 833 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 834 // Concurrent weak refs processing 835 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs); 836 if (heap->gc_cause() == GCCause::_wb_breakpoint) { 837 ShenandoahBreakpoint::at_after_reference_processing_started(); 838 } 839 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */); 840 } 841 842 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure { 843 private: 844 ShenandoahHeap* const _heap; 845 ShenandoahMarkingContext* const _mark_context; 846 bool _evac_in_progress; 847 Thread* const _thread; 848 849 public: 850 ShenandoahEvacUpdateCleanupOopStorageRootsClosure(); 851 void do_oop(oop* p); 852 void do_oop(narrowOop* p); 853 }; 854 855 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() : 856 _heap(ShenandoahHeap::heap()), 857 _mark_context(ShenandoahHeap::heap()->marking_context()), 858 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()), 859 _thread(Thread::current()) { 860 } 861 862 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { 863 const oop obj = RawAccess<>::oop_load(p); 864 if (!CompressedOops::is_null(obj)) { 865 if (!_mark_context->is_marked(obj)) { 866 shenandoah_assert_generations_reconciled(); 867 if (_heap->is_in_active_generation(obj)) { 868 // Here we are asserting that an unmarked from-space object is 'correct'. There seems to be a legitimate 869 // use-case for accessing from-space objects during concurrent class unloading. In all modes of Shenandoah, 870 // concurrent class unloading only happens during a global collection. 871 shenandoah_assert_correct(p, obj); 872 ShenandoahHeap::atomic_clear_oop(p, obj); 873 } 874 } else if (_evac_in_progress && _heap->in_collection_set(obj)) { 875 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 876 if (resolved == obj) { 877 resolved = _heap->evacuate_object(obj, _thread); 878 } 879 shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc()); 880 ShenandoahHeap::atomic_update_oop(resolved, p, obj); 881 } 882 } 883 } 884 885 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) { 886 ShouldNotReachHere(); 887 } 888 889 class ShenandoahIsCLDAliveClosure : public CLDClosure { 890 public: 891 void do_cld(ClassLoaderData* cld) { 892 cld->is_alive(); 893 } 894 }; 895 896 class ShenandoahIsNMethodAliveClosure: public NMethodClosure { 897 public: 898 void do_nmethod(nmethod* n) { 899 n->is_unloading(); 900 } 901 }; 902 903 // This task not only evacuates/updates marked weak roots, but also "null" 904 // dead weak roots. 905 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask { 906 private: 907 ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots; 908 909 // Roots related to concurrent class unloading 910 ShenandoahClassLoaderDataRoots<true /* concurrent */> 911 _cld_roots; 912 ShenandoahConcurrentNMethodIterator _nmethod_itr; 913 ShenandoahPhaseTimings::Phase _phase; 914 915 public: 916 ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 917 WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"), 918 _vm_roots(phase), 919 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 920 _nmethod_itr(ShenandoahCodeRoots::table()), 921 _phase(phase) {} 922 923 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() { 924 // Notify runtime data structures of potentially dead oops 925 _vm_roots.report_num_dead(); 926 } 927 928 void work(uint worker_id) { 929 ShenandoahConcurrentWorkerSession worker_session(worker_id); 930 ShenandoahSuspendibleThreadSetJoiner sts_join; 931 { 932 ShenandoahEvacOOMScope oom; 933 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration 934 // may race against OopStorage::release() calls. 935 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl; 936 _vm_roots.oops_do(&cl, worker_id); 937 } 938 939 // If we are going to perform concurrent class unloading later on, we need to 940 // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we 941 // can cleanup immediate garbage sooner. 942 if (ShenandoahHeap::heap()->unload_classes()) { 943 // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the 944 // CLD's holder or evacuate it. 945 { 946 ShenandoahIsCLDAliveClosure is_cld_alive; 947 _cld_roots.cld_do(&is_cld_alive, worker_id); 948 } 949 950 // Applies ShenandoahIsNMethodAliveClosure to registered nmethods. 951 // The closure calls nmethod->is_unloading(). The is_unloading 952 // state is cached, therefore, during concurrent class unloading phase, 953 // we will not touch the metadata of unloading nmethods 954 { 955 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 956 ShenandoahIsNMethodAliveClosure is_nmethod_alive; 957 _nmethod_itr.nmethods_do(&is_nmethod_alive); 958 } 959 } 960 } 961 }; 962 963 void ShenandoahConcurrentGC::op_weak_roots() { 964 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 965 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); 966 // Concurrent weak root processing 967 { 968 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work); 969 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work); 970 ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work); 971 heap->workers()->run_task(&task); 972 } 973 974 // Perform handshake to flush out dead oops 975 { 976 ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous); 977 heap->rendezvous_threads(); 978 } 979 // We can only toggle concurrent_weak_root_in_progress flag 980 // at a safepoint, so that mutators see a consistent 981 // value. The flag will be cleared at the next safepoint. 982 } 983 984 void ShenandoahConcurrentGC::op_class_unloading() { 985 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 986 assert (heap->is_concurrent_weak_root_in_progress() && 987 heap->unload_classes(), 988 "Checked by caller"); 989 heap->do_class_unloading(); 990 } 991 992 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure { 993 private: 994 BarrierSetNMethod* const _bs; 995 ShenandoahEvacuateUpdateMetadataClosure _cl; 996 997 public: 998 ShenandoahEvacUpdateCodeCacheClosure() : 999 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()), 1000 _cl() { 1001 } 1002 1003 void do_nmethod(nmethod* n) { 1004 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n); 1005 ShenandoahReentrantLocker locker(data->lock()); 1006 // Setup EvacOOM scope below reentrant lock to avoid deadlock with 1007 // nmethod_entry_barrier 1008 ShenandoahEvacOOMScope oom; 1009 data->oops_do(&_cl, true/*fix relocation*/); 1010 _bs->disarm(n); 1011 } 1012 }; 1013 1014 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask { 1015 private: 1016 ShenandoahPhaseTimings::Phase _phase; 1017 ShenandoahVMRoots<true /*concurrent*/> _vm_roots; 1018 ShenandoahClassLoaderDataRoots<true /*concurrent*/> 1019 _cld_roots; 1020 ShenandoahConcurrentNMethodIterator _nmethod_itr; 1021 1022 public: 1023 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : 1024 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"), 1025 _phase(phase), 1026 _vm_roots(phase), 1027 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/), 1028 _nmethod_itr(ShenandoahCodeRoots::table()) {} 1029 1030 void work(uint worker_id) { 1031 ShenandoahConcurrentWorkerSession worker_session(worker_id); 1032 { 1033 ShenandoahEvacOOMScope oom; 1034 { 1035 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration 1036 // may race against OopStorage::release() calls. 1037 ShenandoahContextEvacuateUpdateRootsClosure cl; 1038 _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id); 1039 } 1040 1041 { 1042 ShenandoahEvacuateUpdateMetadataClosure cl; 1043 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong); 1044 _cld_roots.cld_do(&clds, worker_id); 1045 } 1046 } 1047 1048 // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier. 1049 if (!ShenandoahHeap::heap()->unload_classes()) { 1050 ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); 1051 ShenandoahEvacUpdateCodeCacheClosure cl; 1052 _nmethod_itr.nmethods_do(&cl); 1053 } 1054 } 1055 }; 1056 1057 void ShenandoahConcurrentGC::op_strong_roots() { 1058 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1059 assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller"); 1060 ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots); 1061 heap->workers()->run_task(&task); 1062 heap->set_concurrent_strong_root_in_progress(false); 1063 } 1064 1065 void ShenandoahConcurrentGC::op_cleanup_early() { 1066 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1067 } 1068 1069 void ShenandoahConcurrentGC::op_evacuate() { 1070 ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/); 1071 } 1072 1073 void ShenandoahConcurrentGC::op_init_updaterefs() { 1074 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1075 heap->set_evacuation_in_progress(false); 1076 heap->set_concurrent_weak_root_in_progress(false); 1077 heap->prepare_update_heap_references(true /*concurrent*/); 1078 heap->set_update_refs_in_progress(true); 1079 if (ShenandoahVerify) { 1080 heap->verifier()->verify_before_updaterefs(); 1081 } 1082 if (ShenandoahPacing) { 1083 heap->pacer()->setup_for_updaterefs(); 1084 } 1085 } 1086 1087 void ShenandoahConcurrentGC::op_updaterefs() { 1088 ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/); 1089 } 1090 1091 class ShenandoahUpdateThreadClosure : public HandshakeClosure { 1092 private: 1093 ShenandoahUpdateRefsClosure _cl; 1094 public: 1095 ShenandoahUpdateThreadClosure(); 1096 void do_thread(Thread* thread); 1097 }; 1098 1099 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() : 1100 HandshakeClosure("Shenandoah Update Thread Roots") { 1101 } 1102 1103 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) { 1104 if (thread->is_Java_thread()) { 1105 JavaThread* jt = JavaThread::cast(thread); 1106 ResourceMark rm; 1107 jt->oops_do(&_cl, nullptr); 1108 } 1109 } 1110 1111 void ShenandoahConcurrentGC::op_update_thread_roots() { 1112 ShenandoahUpdateThreadClosure cl; 1113 Handshake::execute(&cl); 1114 } 1115 1116 void ShenandoahConcurrentGC::op_final_updaterefs() { 1117 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1118 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); 1119 assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references"); 1120 1121 heap->finish_concurrent_roots(); 1122 1123 // Clear cancelled GC, if set. On cancellation path, the block before would handle 1124 // everything. 1125 if (heap->cancelled_gc()) { 1126 heap->clear_cancelled_gc(true /* clear oom handler */); 1127 } 1128 1129 // Has to be done before cset is clear 1130 if (ShenandoahVerify) { 1131 heap->verifier()->verify_roots_in_to_space(); 1132 } 1133 1134 // If we are running in generational mode and this is an aging cycle, this will also age active 1135 // regions that haven't been used for allocation. 1136 heap->update_heap_region_states(true /*concurrent*/); 1137 1138 heap->set_update_refs_in_progress(false); 1139 heap->set_has_forwarded_objects(false); 1140 1141 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) { 1142 // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to 1143 // objects in the collection set. After those objects are evacuated, the pointers in the 1144 // SATB are no longer safe. Once we have finished update references, we are guaranteed that 1145 // no more writes to the collection set are possible. 1146 // 1147 // This will transfer any old pointers in _active_ regions from the SATB to the old gen 1148 // mark queues. All other pointers will be discarded. This would also discard any pointers 1149 // in old regions that were included in a mixed evacuation. We aren't using the SATB filter 1150 // methods here because we cannot control when they execute. If the SATB filter runs _after_ 1151 // a region has been recycled, we will not be able to detect the bad pointer. 1152 // 1153 // We are not concerned about skipping this step in abbreviated cycles because regions 1154 // with no live objects cannot have been written to and so cannot have entries in the SATB 1155 // buffers. 1156 heap->old_generation()->transfer_pointers_from_satb(); 1157 1158 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for 1159 // entire regions. Both of these relevant operations occur before final update refs. 1160 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false); 1161 } 1162 1163 if (ShenandoahVerify) { 1164 heap->verifier()->verify_after_updaterefs(); 1165 } 1166 1167 if (VerifyAfterGC) { 1168 Universe::verify(); 1169 } 1170 1171 heap->rebuild_free_set(true /*concurrent*/); 1172 } 1173 1174 void ShenandoahConcurrentGC::op_final_roots() { 1175 1176 ShenandoahHeap *heap = ShenandoahHeap::heap(); 1177 heap->set_concurrent_weak_root_in_progress(false); 1178 heap->set_evacuation_in_progress(false); 1179 1180 if (heap->mode()->is_generational()) { 1181 // If the cycle was shortened for having enough immediate garbage, this could be 1182 // the last GC safepoint before concurrent marking of old resumes. We must be sure 1183 // that old mark threads don't see any pointers to garbage in the SATB buffers. 1184 if (heap->is_concurrent_old_mark_in_progress()) { 1185 heap->old_generation()->transfer_pointers_from_satb(); 1186 } 1187 1188 if (!_generation->is_old()) { 1189 ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context()); 1190 } 1191 } 1192 } 1193 1194 void ShenandoahConcurrentGC::op_cleanup_complete() { 1195 ShenandoahHeap::heap()->free_set()->recycle_trash(); 1196 } 1197 1198 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) { 1199 if (ShenandoahHeap::heap()->cancelled_gc()) { 1200 _degen_point = point; 1201 return true; 1202 } 1203 return false; 1204 } 1205 1206 const char* ShenandoahConcurrentGC::init_mark_event_message() const { 1207 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1208 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); 1209 if (heap->unload_classes()) { 1210 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)"); 1211 } else { 1212 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", ""); 1213 } 1214 } 1215 1216 const char* ShenandoahConcurrentGC::final_mark_event_message() const { 1217 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1218 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1219 "Should not have forwarded objects during final mark, unless old gen concurrent mark is running"); 1220 1221 if (heap->unload_classes()) { 1222 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)"); 1223 } else { 1224 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", ""); 1225 } 1226 } 1227 1228 const char* ShenandoahConcurrentGC::conc_mark_event_message() const { 1229 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 1230 assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), 1231 "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running"); 1232 if (heap->unload_classes()) { 1233 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)"); 1234 } else { 1235 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", ""); 1236 } 1237 } 1238 1239 const char* ShenandoahConcurrentGC::conc_reset_event_message() const { 1240 if (ShenandoahHeap::heap()->unload_classes()) { 1241 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)"); 1242 } else { 1243 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", ""); 1244 } 1245 } 1246 1247 const char* ShenandoahConcurrentGC::final_roots_event_message() const { 1248 if (ShenandoahHeap::heap()->unload_classes()) { 1249 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)"); 1250 } else { 1251 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", ""); 1252 } 1253 } 1254 1255 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const { 1256 if (ShenandoahHeap::heap()->unload_classes()) { 1257 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)"); 1258 } else { 1259 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", ""); 1260 } 1261 } 1262 1263 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const { 1264 if (ShenandoahHeap::heap()->unload_classes()) { 1265 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)"); 1266 } else { 1267 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", ""); 1268 } 1269 } 1270 1271 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const { 1272 if (ShenandoahHeap::heap()->unload_classes()) { 1273 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)"); 1274 } else { 1275 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", ""); 1276 } 1277 }