src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

*** 71,10 ***
--- 71,12 ---
  #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+ #include "gc/shenandoah/shenandoahStackChunkGCData.inline.hpp"
+ #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  #include "gc/shenandoah/shenandoahSTWMark.hpp"
  #include "gc/shenandoah/shenandoahUncommitThread.hpp"
  #include "gc/shenandoah/shenandoahUtils.hpp"
  #include "gc/shenandoah/shenandoahVerifier.hpp"
  #include "gc/shenandoah/shenandoahVMOperations.hpp"

*** 1227,11 ***
--- 1229,28 ---
    assert(generation->is_global(), "Only global generation expected here");
    ShenandoahEvacuationTask task(this, _collection_set, concurrent);
    workers()->run_task(&task);
  }
  
+ class ShenandoahCompleteStackWatermarkHandshakeClosure : public HandshakeClosure {
+ public:
+   ShenandoahCompleteStackWatermarkHandshakeClosure() : HandshakeClosure("Shenandoah Complete stacks handshake") {}
+   void do_thread(Thread* thread) override {
+     if (thread->is_Java_thread()) {
+       JavaThread* jt = JavaThread::cast(thread);
+       StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
+     }
+   }
+ };
+ 
  void ShenandoahHeap::concurrent_prepare_for_update_refs() {
+   // The stack watermarks are active since op_final_roots().
+   // Make sure the current stack watermark machinery has completed before we drop evac flags.
+   // Otherwise the stack processing on stack unwinding may enter evac closure concurrently.
+   ShenandoahCompleteStackWatermarkHandshakeClosure cl;
+   Handshake::execute(&cl);
+ 
    {
      // Java threads take this lock while they are being attached and added to the list of threads.
      // If another thread holds this lock before we update the gc state, it will receive a stale
      // gc state, but they will have been added to the list of java threads and so will be corrected
      // by the following handshake.

*** 1252,28 ***
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! void ShenandoahHeap::concurrent_final_roots() {
-   {
-     MutexLocker lock(Threads_lock);
- 
  #ifdef ASSERT
      for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
        StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
        assert(sw == nullptr || sw->processing_completed(),
               "Cannot turn off weak roots before stack watermark processing is complete");
      }
  #endif
  
!     set_gc_state_concurrent(WEAK_ROOTS, false);
!   }
  
!   ShenandoahGCStatePropagatorHandshakeClosure propagator(_gc_state.raw_value());
!   Threads::non_java_threads_do(&propagator);
!   Handshake::execute(&propagator);
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
  
--- 1271,36 ---
    Handshake::execute(&prepare_for_update_refs);
  
    _update_refs_iterator.reset();
  }
  
! void ShenandoahHeap::op_final_roots() {
  #ifdef ASSERT
+   // Check if stack watermark machinery is in safe state:
+   //  1. With evac-in-progress, we are about to supersede evac processing with new epoch.
+   //     op_thread_roots() should have completed the stack watermark processing before
+   //     we reach here.
+   //  2. Without evac-in-progress, we are in abbreviated cycle, and we are switching
+   //     to a new epoch that *also* does not change any oops, which is safe.
+   if (is_evacuation_in_progress()) {
      for (JavaThreadIteratorWithHandle jtiwh; JavaThread* jt = jtiwh.next();) {
        StackWatermark* sw = StackWatermarkSet::get(jt, StackWatermarkKind::gc);
        assert(sw == nullptr || sw->processing_completed(),
               "Cannot turn off weak roots before stack watermark processing is complete");
      }
+   }
  #endif
  
!   set_gc_state_at_safepoint(WEAK_ROOTS, false);
! 
+   // Arm the nmethods to change the barriers.
+   CodeCache::arm_all_nmethods();
  
!   {
!     ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
!     propagate_gc_state_to_all_threads();
+   }
  }
  
  oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
    assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
  

*** 2896,21 ***
      }
    }
  }
  
  bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
!   if (is_idle()) return false;
  
!   // Objects allocated after marking start are implicitly alive, don't need any barriers during
-   // marking phase.
    if (is_concurrent_mark_in_progress() &&
!      !marking_context()->allocated_after_mark_start(obj)) {
      return true;
    }
  
!   // Can not guarantee obj is deeply good.
!   if (has_forwarded_objects()) {
      return true;
    }
  
    return false;
  }
--- 2923,23 ---
      }
    }
  }
  
  bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
!   // Can not guarantee obj is deeply good.
+   if (has_forwarded_objects()) {
+     return true;
+   }
  
!   // Objects allocated after marking start are implicitly alive.
    if (is_concurrent_mark_in_progress() &&
!       !marking_context()->allocated_after_mark_start(obj)) {
      return true;
    }
  
!   // Nmethods referenced by stack chunk may need the barrier fixups.
!   if (ShenandoahStackChunkGCData::is_different_epoch(obj)) {
      return true;
    }
  
    return false;
  }