src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp

@@ -26,26 +26,30 @@
  #define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
  
  #include "gc/shenandoah/shenandoahForwarding.hpp"
  
  #include "gc/shenandoah/shenandoahAsserts.hpp"
+ #include "oops/klass.hpp"
  #include "oops/markWord.hpp"
  #include "runtime/javaThread.hpp"
  
  inline oop ShenandoahForwarding::get_forwardee_raw(oop obj) {
    shenandoah_assert_in_heap_bounds(nullptr, obj);
    return get_forwardee_raw_unchecked(obj);
  }
  
+ static HeapWord* to_forwardee(markWord mark) {
+   return reinterpret_cast<HeapWord*>(mark.value() & ~(markWord::lock_mask_in_place | markWord::self_fwd_mask_in_place));
+ }
  inline oop ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
    // JVMTI and JFR code use mark words for marking objects for their needs.
    // On this path, we can encounter the "marked" object, but with null
    // fwdptr. That object is still not forwarded, and we need to return
    // the object itself.
    markWord mark = obj->mark();
    if (mark.is_marked()) {
-     HeapWord* fwdptr = (HeapWord*) mark.clear_lock_bits().to_pointer();
+     HeapWord* fwdptr = to_forwardee(mark);
      if (fwdptr != nullptr) {
        return cast_to_oop(fwdptr);
      }
    }
    return obj;

@@ -56,11 +60,11 @@
    shenandoah_assert_correct(nullptr, obj);
    assert(Thread::current()->is_Java_thread(), "Must be a mutator thread");
  
    markWord mark = obj->mark();
    if (mark.is_marked()) {
-     HeapWord* fwdptr = (HeapWord*) mark.clear_lock_bits().to_pointer();
+     HeapWord* fwdptr = to_forwardee(mark);
      assert(fwdptr != nullptr, "Forwarding pointer is never null here");
      return cast_to_oop(fwdptr);
    } else {
      return obj;
    }

@@ -69,42 +73,66 @@
  inline oop ShenandoahForwarding::get_forwardee(oop obj) {
    shenandoah_assert_correct(nullptr, obj);
    return get_forwardee_raw_unchecked(obj);
  }
  
+ inline bool ShenandoahForwarding::is_forwarded(markWord m) {
+   return (m.value() & (markWord::lock_mask_in_place | markWord::self_fwd_mask_in_place)) > markWord::monitor_value;
+ }
+ 
  inline bool ShenandoahForwarding::is_forwarded(oop obj) {
-   return obj->mark().is_marked();
+   return is_forwarded(obj->mark());
  }
  
  inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
    markWord old_mark = obj->mark();
-   if (old_mark.is_marked()) {
-     return cast_to_oop(old_mark.clear_lock_bits().to_pointer());
+   if (is_forwarded(old_mark)) {
+     return cast_to_oop(to_forwardee(old_mark));
    }
  
    markWord new_mark = markWord::encode_pointer_as_mark(update);
+   if (UseCompactObjectHeaders && old_mark.is_hashed_not_expanded()) {
+     new_mark = markWord(new_mark.value() | FWDED_HASH_TRANSITION);
+   }
    markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
    if (prev_mark == old_mark) {
      return update;
    } else {
-     return cast_to_oop(prev_mark.clear_lock_bits().to_pointer());
+     return cast_to_oop(to_forwardee(prev_mark));
    }
  }
  
  inline Klass* ShenandoahForwarding::klass(oop obj) {
    if (UseCompactObjectHeaders) {
      markWord mark = obj->mark();
      if (mark.is_marked()) {
-       oop fwd = cast_to_oop(mark.clear_lock_bits().to_pointer());
+       oop fwd = cast_to_oop(to_forwardee(mark));
        mark = fwd->mark();
      }
      return mark.klass();
    } else {
      return obj->klass();
    }
  }
  
  inline size_t ShenandoahForwarding::size(oop obj) {
-   return obj->size_given_klass(klass(obj));
+   markWord mark = obj->mark();
+   if (is_forwarded(mark)) {
+     oop fwd = cast_to_oop(to_forwardee(mark));
+     markWord fwd_mark = fwd->mark();
+     Klass* klass = UseCompactObjectHeaders ? fwd_mark.klass() : fwd->klass();
+     size_t size = fwd->base_size_given_klass(fwd_mark, klass);
+     if (UseCompactObjectHeaders) {
+       if ((mark.value() & FWDED_HASH_TRANSITION) != FWDED_HASH_TRANSITION) {
+         if (fwd_mark.is_expanded() && klass->expand_for_hash(fwd)) {
+           size = align_object_size(size + 1);
+         }
+       }
+     }
+     return size;
+   } else {
+     Klass* klass = UseCompactObjectHeaders ? mark.klass() : obj->klass();
+     return obj->size_given_mark_and_klass(mark, klass);
+   }
  }
  
  #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP