src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -71,10 +71,11 @@
  #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+ #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  #include "gc/shenandoah/shenandoahSTWMark.hpp"
  #include "gc/shenandoah/shenandoahUncommitThread.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
  #include "gc/shenandoah/shenandoahVerifier.hpp"
  #include "gc/shenandoah/shenandoahVMOperations.hpp"

@@ -1225,11 +1226,28 @@
    assert(generation->is_global(), "Only global generation expected here");
    ShenandoahEvacuationTask task(this, _collection_set, concurrent);
    workers()->run_task(&task);
  }
  
+ class ShenandoahCompleteStackWatermarkHandshakeClosure : public HandshakeClosure {
+ public:
+   ShenandoahCompleteStackWatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
+   void do_thread(Thread* thread) override {
+     if (thread->is_Java_thread()) {
+       JavaThread* jt = JavaThread::cast(thread);
+       StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
+     }
+   }
+ };
+ 
  void ShenandoahHeap::concurrent_prepare_for_update_refs() {
+   // The stack watermarks are active since op_final_roots().
+   // Make sure the current stack watermark machinery has completed before we drop evac flags.
+   // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
+   ShenandoahCompleteStackWatermarkHandshakeClosure cl;
+   Handshake::execute(&cl);
+ 
    {
      // Java threads take this lock while they are being attached and added to the list of threads.
      // If another thread holds this lock before we update the gc state, it will receive a stale
      // gc state, but they will have been added to the list of java threads and so will be corrected
      // by the following handshake.

@@ -1250,28 +1268,36 @@
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
- void ShenandoahHeap::concurrent_final_roots() {
-   {
-     MutexLocker lock(Threads_lock);
- 
+ void ShenandoahHeap::op_final_roots() {
  #ifdef ASSERT
+   // Check if stack watermark machinery is in safe state:
+   //  1. With evac-in-progress, we are about to supersede evac processing with new epoch.
+   //     op_thread_roots() should have completed the stack watermark processing before
+   //     we reach here.
+   //  2. Without evac-in-progress, we are in abbreviated cycle, and we are switching
+   //     to a new epoch that *also* does not change any oops, which is safe.
+   if (is_evacuation_in_progress()) {
      for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
        StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
        assert(sw == nullptr || sw->processing_completed(),
               "Cannot turn off weak roots before stack watermark processing is complete");
      }
+   }
  #endif
  
-     set_gc_state_concurrent(WEAK_ROOTS, false);
-   }
+   set_gc_state_at_safepoint(WEAK_ROOTS, false);
  
-   ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
-   Threads::non_java_threads_do(&propagator);
-   Handshake::execute(&propagator);
+   // Arm the nmethods to change the barriers.
+   ShenandoahCodeRoots::arm_nmethods();
+ 
+   {
+     ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
+     propagate_gc_state_to_all_threads();
+   }
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");