src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -26,11 +26,11 @@
  
  
  #include "cds/aotMappedHeapWriter.hpp"
  #include "classfile/systemDictionary.hpp"
  #include "gc/shared/classUnloadingContext.hpp"
- #include "gc/shared/fullGCForwarding.hpp"
+ #include "gc/shared/fullGCForwarding.inline.hpp"
  #include "gc/shared/gc_globals.hpp"
  #include "gc/shared/gcArguments.hpp"
  #include "gc/shared/gcTimer.hpp"
  #include "gc/shared/gcTraceTime.inline.hpp"
  #include "gc/shared/locationPrinter.inline.hpp"

@@ -1312,11 +1312,17 @@
                                                 ShenandoahAffiliation target_gen) {
    assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
    assert(from_region->is_young(), "Only expect evacuations from young in this mode");
    bool alloc_from_lab = true;
    HeapWord* copy = nullptr;
-   size_t size = ShenandoahForwarding::size(p);
+ 
+   markWord mark = p->mark();
+   if (ShenandoahForwarding::is_forwarded(mark)) {
+     return ShenandoahForwarding::get_forwardee(p);
+   }
+   size_t old_size = ShenandoahForwarding::size(p);
+   size_t size = p->copy_size(old_size, mark);
  
  #ifdef ASSERT
    if (ShenandoahOOMDuringEvacALot &&
        (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
      copy = nullptr;

@@ -1359,14 +1365,23 @@
    if (ShenandoahEvacTracking) {
      evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen);
    }
  
    // Copy the object:
-   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
- 
+   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
    oop copy_val = cast_to_oop(copy);
  
+   // Initialize the identity hash on the copy before installing the forwarding
+   // pointer, using the mark word we captured earlier. We must do this before
+   // the CAS so that the copy is fully initialized when it becomes visible to
+   // other threads. Using the captured mark (rather than re-reading the copy's
+   // mark) avoids races with other threads that may have evacuated p and
+   // installed a forwarding pointer in the meantime.
+   if (UseCompactObjectHeaders && mark.is_hashed_not_expanded()) {
+     copy_val->set_mark(copy_val->initialize_hash_if_necessary(p, mark.klass(), mark));
+   }
+ 
    // Relativize stack chunks before publishing the copy. After the forwarding CAS,
    // mutators can see the copy and thaw it via the fast path if flags == 0. We must
    // relativize derived pointers and set gc_mode before that happens. Skip if the
    // copy's mark word is already a forwarding pointer (another thread won the race
    // and overwrote the original's header before we copied it).