src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

*** 69,10 ***
--- 69,11 ---
  #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+ #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  #include "gc/shenandoah/shenandoahSTWMark.hpp"
  #include "gc/shenandoah/shenandoahUncommitThread.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
  #include "gc/shenandoah/shenandoahVerifier.hpp"
  #include "gc/shenandoah/shenandoahVMOperations.hpp"

*** 1226,21 ***
    assert(generation->is_global(), "Only global generation expected here");
    ShenandoahEvacuationTask task(this, _collection_set, concurrent);
    workers()->run_task(&task);
  }
  
  void ShenandoahHeap::concurrent_prepare_for_update_refs() {
    {
      // Java threads take this lock while they are being attached and added to the list of threads.
      // If another thread holds this lock before we update the gc state, it will receive a stale
      // gc state, but they will have been added to the list of java threads and so will be corrected
      // by the following handshake.
      MutexLocker lock(Threads_lock);
  
      // A cancellation at this point means the degenerated cycle must resume from update-refs.
      set_gc_state_concurrent(EVACUATION, false);
-     set_gc_state_concurrent(WEAK_ROOTS, false);
      set_gc_state_concurrent(UPDATE_REFS, true);
    }
  
    // This will propagate the gc state and retire gclabs and plabs for threads that require it.
    ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
--- 1227,36 ---
    assert(generation->is_global(), "Only global generation expected here");
    ShenandoahEvacuationTask task(this, _collection_set, concurrent);
    workers()->run_task(&task);
  }
  
+ class ShenandoahCompleteStackwatermarkHandshakeClosure : public HandshakeClosure {
+ public:
+   ShenandoahCompleteStackwatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
+   void do_thread(Thread* thread) override {
+     if (thread->is_Java_thread()) {
+       JavaThread* jt = JavaThread::cast(thread);
+       StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
+     }
+   }
+ };
+ 
  void ShenandoahHeap::concurrent_prepare_for_update_refs() {
+   // Make sure the current stack watermark machinery has completed before we drop evac flags.
+   // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
+   ShenandoahCompleteStackwatermarkHandshakeClosure cl;
+   Handshake::execute(&cl);
+ 
    {
      // Java threads take this lock while they are being attached and added to the list of threads.
      // If another thread holds this lock before we update the gc state, it will receive a stale
      // gc state, but they will have been added to the list of java threads and so will be corrected
      // by the following handshake.
      MutexLocker lock(Threads_lock);
  
      // A cancellation at this point means the degenerated cycle must resume from update-refs.
      set_gc_state_concurrent(EVACUATION, false);
      set_gc_state_concurrent(UPDATE_REFS, true);
    }
  
    // This will propagate the gc state and retire gclabs and plabs for threads that require it.
    ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());

*** 1252,39 ***
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! class ShenandoahCompositeHandshakeClosure : public HandshakeClosure {
!   HandshakeClosure* _handshake_1;
!   HandshakeClosure* _handshake_2;
-   public:
-     ShenandoahCompositeHandshakeClosure(HandshakeClosure* handshake_1, HandshakeClosure* handshake_2) :
-       HandshakeClosure(handshake_2->name()),
-       _handshake_1(handshake_1), _handshake_2(handshake_2) {}
  
!   void do_thread(Thread* thread) override {
!       _handshake_1->do_thread(thread);
-       _handshake_2->do_thread(thread);
-     }
- };
- 
- void ShenandoahHeap::concurrent_final_roots(HandshakeClosure* handshake_closure) {
-   {
-     assert(!is_evacuation_in_progress(), "Should not evacuate for abbreviated or old cycles");
-     MutexLocker lock(Threads_lock);
-     set_gc_state_concurrent(WEAK_ROOTS, false);
    }
  
!   ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
!   Threads::non_java_threads_do(&propagator);
!   if (handshake_closure == nullptr) {
-     Handshake::execute(&propagator);
-   } else {
-     ShenandoahCompositeHandshakeClosure composite(&propagator, handshake_closure);
-     Handshake::execute(&composite);
-   }
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
  
--- 1268,21 ---
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! void ShenandoahHeap::op_final_roots(bool at_gc_end) {
!   set_gc_state_at_safepoint(WEAK_ROOTS, false);
!   propagate_gc_state_to_all_threads();
  
!   if (ShenandoahVerify && at_gc_end) {
!     verifier()->verify_after_gc(active_generation());
    }
  
!   // Arm the nmethods to put barriers down.
!   ShenandoahCodeRoots::arm_nmethods();
!   ShenandoahStackWatermark::change_epoch_id();
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");