src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

*** 71,10 ***
--- 71,11 ---
  #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+ #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  #include "gc/shenandoah/shenandoahSTWMark.hpp"
  #include "gc/shenandoah/shenandoahUncommitThread.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
  #include "gc/shenandoah/shenandoahVerifier.hpp"
  #include "gc/shenandoah/shenandoahVMOperations.hpp"

*** 1225,11 ***
--- 1226,27 ---
    assert(generation->is_global(), "Only global generation expected here");
    ShenandoahEvacuationTask task(this, _collection_set, concurrent);
    workers()->run_task(&task);
  }
  
+ class ShenandoahCompleteStackwatermarkHandshakeClosure : public HandshakeClosure {
+ public:
+   ShenandoahCompleteStackwatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
+   void do_thread(Thread* thread) override {
+     if (thread->is_Java_thread()) {
+       JavaThread* jt = JavaThread::cast(thread);
+       StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
+     }
+   }
+ };
+ 
  void ShenandoahHeap::concurrent_prepare_for_update_refs() {
+   // Make sure the current stack watermark machinery has completed before we drop evac flags.
+   // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
+   ShenandoahCompleteStackwatermarkHandshakeClosure cl;
+   Handshake::execute(&cl);
+ 
    {
      // Java threads take this lock while they are being attached and added to the list of threads.
      // If another thread holds this lock before we update the gc state, it will receive a stale
      // gc state, but they will have been added to the list of java threads and so will be corrected
      // by the following handshake.

*** 1250,30 ***
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! void ShenandoahHeap::concurrent_final_roots() {
    {
      MutexLocker lock(Threads_lock);
  
- #ifdef ASSERT
-     for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
-       StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
-       assert(sw == nullptr || sw->processing_completed(),
-              "Cannot turn off weak roots before stack watermark processing is complete");
-     }
- #endif
- 
      set_gc_state_concurrent(WEAK_ROOTS, false);
    }
  
    ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
    Threads::non_java_threads_do(&propagator);
    Handshake::execute(&propagator);
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
  
    ShenandoahHeapRegion* r = heap_region_containing(p);
    assert(!r->is_humongous(), "never evacuate humongous objects");
--- 1267,36 ---
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! void ShenandoahHeap::op_conc_roots() {
+   // Make sure the current stack watermark machinery has completed before we drop flags.
+   ShenandoahCompleteStackwatermarkHandshakeClosure cl;
+   Handshake::execute(&cl);
+ 
    {
+     // Java threads take this lock while they are being attached and added to the list of threads.
+     // If another thread holds this lock before we update the gc state, it will receive a stale
+     // gc state, but they will have been added to the list of java threads and so will be corrected
+     // by the following handshake.
      MutexLocker lock(Threads_lock);
  
      set_gc_state_concurrent(WEAK_ROOTS, false);
    }
  
    ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
    Threads::non_java_threads_do(&propagator);
    Handshake::execute(&propagator);
  }
  
+ void ShenandoahHeap::op_final_roots() {
+   // Arm the nmethods to put barriers down.
+   ShenandoahCodeRoots::arm_nmethods();
+   ShenandoahStackWatermark::change_epoch_id();
+ }
+ 
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
  
    ShenandoahHeapRegion* r = heap_region_containing(p);
    assert(!r->is_humongous(), "never evacuate humongous objects");