src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

*** 212,11 ***
    if (heap->is_concurrent_strong_root_in_progress()) {
      entry_strong_roots();
    }
  
    // Roots processing is complete, put the weak roots flag down.
!   entry_final_roots();
  
    // Continue the cycle with evacuation and optional update-refs.
    // This may be skipped if there is nothing to evacuate.
    // If so, evac_in_progress would be unset by collection set preparation code.
    if (heap->is_evacuation_in_progress()) {
--- 212,11 ---
    if (heap->is_concurrent_strong_root_in_progress()) {
      entry_strong_roots();
    }
  
    // Roots processing is complete, put the weak roots flag down.
!   vmop_entry_final_roots();
  
    // Continue the cycle with evacuation and optional update-refs.
    // This may be skipped if there is nothing to evacuate.
    // If so, evac_in_progress would be unset by collection set preparation code.
    if (heap->is_evacuation_in_progress()) {

*** 361,10 ***
--- 361,21 ---
    heap->try_inject_pin();
    VM_ShenandoahFinalUpdateRefs op(this);
    VMThread::execute(&op);
  }
  
+ void ShenandoahConcurrentGC::vmop_entry_final_roots() {
+   ShenandoahHeap* const heap = ShenandoahHeap::heap();
+   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
+   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
+ 
+   // This phase does not use workers, no need for setup
+   heap->try_inject_alloc_failure();
+   VM_ShenandoahFinalRoots op(this);
+   VMThread::execute(&op);
+ }
+ 
  void ShenandoahConcurrentGC::vmop_entry_final_verify() {
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
    ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
  

*** 765,11 ***
    // Make above changes visible to worker threads
    OrderAccess::fence();
  
    // Arm nmethods/stack for concurrent processing
    ShenandoahCodeRoots::arm_nmethods();
-   ShenandoahStackWatermark::change_epoch_id();
  
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
--- 776,10 ---

*** 821,15 ***
        }
  
        heap->set_evacuation_in_progress(true);
        // From here on, we need to update references.
        heap->set_has_forwarded_objects(true);
- 
-       // Arm nmethods/stack for concurrent processing
-       ShenandoahCodeRoots::arm_nmethods();
-       ShenandoahStackWatermark::change_epoch_id();
- 
      } else {
        if (ShenandoahVerify) {
          ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
          if (has_in_place_promotions(heap)) {
            heap->verifier()->verify_after_concmark_with_promotions(_generation);
--- 831,10 ---

*** 838,10 ***
--- 843,13 ---
          }
        }
      }
    }
  
+   // Arm nmethods/stack for concurrent processing
+   ShenandoahCodeRoots::arm_nmethods();
+ 
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }

*** 1058,11 ***
  
    void do_nmethod(nmethod* n) {
      ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
      ShenandoahNMethodLocker locker(data->lock());
      data->oops_do(&_cl, /* fix_relocations = */ true);
-     ShenandoahNMethod::disarm_nmethod(n);
    }
  };
  
  class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
  private:
--- 1066,10 ---

*** 1239,24 ***
    }
  
    heap->rebuild_free_set(true /*concurrent*/);
    _generation->heuristics()->start_idle_span();
  
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }
  
  void ShenandoahConcurrentGC::entry_final_roots() {
!   ShenandoahHeap* const heap = ShenandoahHeap::heap();
!   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
-   const char* msg = conc_final_roots_event_message();
-   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
    EventMark em("%s", msg);
  
!   heap->concurrent_final_roots();
  }
  
  void ShenandoahConcurrentGC::op_verify_final() {
    assert(ShenandoahVerify, "Should have been checked before");
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
--- 1246,25 ---
    }
  
    heap->rebuild_free_set(true /*concurrent*/);
    _generation->heuristics()->start_idle_span();
  
+   // Final pause: update GC barriers to idle state.
+   ShenandoahCodeRoots::arm_nmethods();
+ 
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }
  
  void ShenandoahConcurrentGC::entry_final_roots() {
!   const char* msg = final_roots_event_message();
!   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
    EventMark em("%s", msg);
  
!   ShenandoahHeap::heap()->op_final_roots();
  }
  
  void ShenandoahConcurrentGC::op_verify_final() {
    assert(ShenandoahVerify, "Should have been checked before");
    ShenandoahHeap* const heap = ShenandoahHeap::heap();

*** 1351,15 ***
    } else {
      SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
    }
  }
  
! const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
    if (ShenandoahHeap::heap()->unload_classes()) {
!     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
    } else {
!     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
    }
  }
  
  const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
    if (ShenandoahHeap::heap()->unload_classes()) {
--- 1359,15 ---
    } else {
      SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
    }
  }
  
! const char* ShenandoahConcurrentGC::final_roots_event_message() const {
    if (ShenandoahHeap::heap()->unload_classes()) {
!     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
    } else {
!     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
    }
  }
  
  const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
    if (ShenandoahHeap::heap()->unload_classes()) {