src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

*** 212,11 ***
    if (heap->is_concurrent_strong_root_in_progress()) {
      entry_strong_roots();
    }
  
    // Roots processing is complete, put the weak roots flag down.
!   entry_final_roots();
  
    // Continue the cycle with evacuation and optional update-refs.
    // This may be skipped if there is nothing to evacuate.
    // If so, evac_in_progress would be unset by collection set preparation code.
    if (heap->is_evacuation_in_progress()) {
--- 212,12 ---
    if (heap->is_concurrent_strong_root_in_progress()) {
      entry_strong_roots();
    }
  
    // Roots processing is complete, put the weak roots flag down.
!   entry_conc_roots();
+   vmop_entry_final_roots();
  
    // Continue the cycle with evacuation and optional update-refs.
    // This may be skipped if there is nothing to evacuate.
    // If so, evac_in_progress would be unset by collection set preparation code.
    if (heap->is_evacuation_in_progress()) {

*** 361,10 ***
--- 362,21 ---
    heap->try_inject_pin();
    VM_ShenandoahFinalUpdateRefs op(this);
    VMThread::execute(&op);
  }
  
+ void ShenandoahConcurrentGC::vmop_entry_final_roots() {
+   ShenandoahHeap* const heap = ShenandoahHeap::heap();
+   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
+   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
+ 
+   // This phase does not use workers, no need for setup
+   heap->try_inject_alloc_failure();
+   VM_ShenandoahFinalRoots op(this);
+   VMThread::execute(&op);
+ }
+ 
  void ShenandoahConcurrentGC::vmop_entry_final_verify() {
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
    TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
    ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
  

*** 821,15 ***
        }
  
        heap->set_evacuation_in_progress(true);
        // From here on, we need to update references.
        heap->set_has_forwarded_objects(true);
- 
-       // Arm nmethods/stack for concurrent processing
-       ShenandoahCodeRoots::arm_nmethods();
-       ShenandoahStackWatermark::change_epoch_id();
- 
      } else {
        if (ShenandoahVerify) {
          ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
          if (has_in_place_promotions(heap)) {
            heap->verifier()->verify_after_concmark_with_promotions(_generation);
--- 833,10 ---

*** 838,10 ***
--- 845,14 ---
          }
        }
      }
    }
  
+   // Arm nmethods/stack for concurrent processing
+   ShenandoahCodeRoots::arm_nmethods();
+   ShenandoahStackWatermark::change_epoch_id();
+ 
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }

*** 1058,11 ***
  
    void do_nmethod(nmethod* n) {
      ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
      ShenandoahNMethodLocker locker(data->lock());
      data->oops_do(&_cl, /* fix_relocations = */ true);
-     ShenandoahNMethod::disarm_nmethod(n);
    }
  };
  
  class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
  private:
--- 1069,10 ---

*** 1239,24 ***
    }
  
    heap->rebuild_free_set(true /*concurrent*/);
    _generation->heuristics()->start_idle_span();
  
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }
  
! void ShenandoahConcurrentGC::entry_final_roots() {
-   ShenandoahHeap* const heap = ShenandoahHeap::heap();
-   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
    const char* msg = conc_final_roots_event_message();
!   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
    EventMark em("%s", msg);
  
!   heap->concurrent_final_roots();
  }
  
  void ShenandoahConcurrentGC::op_verify_final() {
    assert(ShenandoahVerify, "Should have been checked before");
    ShenandoahHeap* const heap = ShenandoahHeap::heap();
--- 1249,36 ---
    }
  
    heap->rebuild_free_set(true /*concurrent*/);
    _generation->heuristics()->start_idle_span();
  
+   {
+     // Final pause: update GC barriers to idle state.
+     ShenandoahCodeRoots::arm_nmethods();
+     ShenandoahStackWatermark::change_epoch_id();
+   }
+ 
    {
      ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
      heap->propagate_gc_state_to_all_threads();
    }
  }
  
! void ShenandoahConcurrentGC::entry_conc_roots() {
    const char* msg = conc_final_roots_event_message();
!   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_roots);
+   EventMark em("%s", msg);
+ 
+   ShenandoahHeap::heap()->op_conc_roots();
+ }
+ 
+ void ShenandoahConcurrentGC::entry_final_roots() {
+   const char* msg = "Pause Final Roots";
+   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
    EventMark em("%s", msg);
  
!   ShenandoahHeap::heap()->op_final_roots();
  }
  
  void ShenandoahConcurrentGC::op_verify_final() {
    assert(ShenandoahVerify, "Should have been checked before");
    ShenandoahHeap* const heap = ShenandoahHeap::heap();