src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

*** 32,10 ***
--- 32,11 ---
  #include "gc/shared/gcTimer.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
  #include "gc/shared/locationPrinter.inline.hpp"
  #include "gc/shared/memAllocator.hpp"
  #include "gc/shared/plab.hpp"
+ #include "gc/shared/slidingForwarding.hpp"
  #include "gc/shared/tlab_globals.hpp"
  
  #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahCollectionSet.hpp"

*** 437,10 ***
--- 438,12 ---
  
    _control_thread = new ShenandoahControlThread();
  
    ShenandoahInitLogger::print();
  
+   SlidingForwarding::initialize(_heap_region, ShenandoahHeapRegion::region_size_words());
+ 
    return JNI_OK;
  }
  
  void ShenandoahHeap::initialize_mode() {
    if (ShenandoahGCMode != nullptr) {

*** 1126,11 ***
      return ShenandoahBarrierSet::resolve_forwarded(p);
    }
  
    assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
  
!   size_t size = p->size();
  
    assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
  
    bool alloc_from_gclab = true;
    HeapWord* copy = nullptr;
--- 1129,11 ---
      return ShenandoahBarrierSet::resolve_forwarded(p);
    }
  
    assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
  
!   size_t size = p->forward_safe_size();
  
    assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
  
    bool alloc_from_gclab = true;
    HeapWord* copy = nullptr;

*** 1161,15 ***
      return ShenandoahBarrierSet::resolve_forwarded(p);
    }
  
    // Copy the object:
    Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
- 
-   // Try to install the new forwarding pointer.
    oop copy_val = cast_to_oop(copy);
-   ContinuationGCSupport::relativize_stack_chunk(copy_val);
  
    oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
    if (result == copy_val) {
      // Successfully evacuated. Our copy is now the public one!
      shenandoah_assert_correct(nullptr, copy_val);
      return copy_val;
--- 1164,29 ---
      return ShenandoahBarrierSet::resolve_forwarded(p);
    }
  
    // Copy the object:
    Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
    oop copy_val = cast_to_oop(copy);
  
+   if (UseCompactObjectHeaders) {
+     // The copy above is not atomic. Make sure we have seen the proper mark
+     // and re-install it into the copy, so that Klass* is guaranteed to be correct.
+     markWord mark = copy_val->mark();
+     if (!mark.is_marked()) {
+       copy_val->set_mark(mark);
+       ContinuationGCSupport::relativize_stack_chunk(copy_val);
+     } else {
+       // If we copied a mark-word that indicates 'forwarded' state, the object
+       // installation would not succeed. We cannot access Klass* anymore either.
+       // Skip the transformation.
+     }
+   } else {
+     ContinuationGCSupport::relativize_stack_chunk(copy_val);
+   }
+ 
+   // Try to install the new forwarding pointer.
    oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
    if (result == copy_val) {
      // Successfully evacuated. Our copy is now the public one!
      shenandoah_assert_correct(nullptr, copy_val);
      return copy_val;