src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

*** 96,11 ***
    template <class T>
    void do_oop_work(T* p) {
      T o = RawAccess<>::oop_load(p);
      if (!CompressedOops::is_null(o)) {
        oop obj = CompressedOops::decode_not_null(o);
!       if (is_instance_ref_klass(obj->klass())) {
          obj = ShenandoahForwarding::get_forwardee(obj);
        }
        // Single threaded verification can use faster non-atomic stack and bitmap
        // methods.
        //
--- 96,11 ---
    template <class T>
    void do_oop_work(T* p) {
      T o = RawAccess<>::oop_load(p);
      if (!CompressedOops::is_null(o)) {
        oop obj = CompressedOops::decode_not_null(o);
!       if (is_instance_ref_klass(obj->forward_safe_klass())) {
          obj = ShenandoahForwarding::get_forwardee(obj);
        }
        // Single threaded verification can use faster non-atomic stack and bitmap
        // methods.
        //

*** 123,11 ***
                "oop must be in heap");
      check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
                "oop must be aligned");
  
      ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
!     Klass* obj_klass = obj->klass_or_null();
  
      // Verify that obj is not in dead space:
      {
        // Do this before touching obj->size()
        check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,
--- 123,11 ---
                "oop must be in heap");
      check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
                "oop must be aligned");
  
      ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
!     Klass* obj_klass = obj->forward_safe_klass();
  
      // Verify that obj is not in dead space:
      {
        // Do this before touching obj->size()
        check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != nullptr,

*** 138,15 ***
        HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
        check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
               "Object start should be within the region");
  
        if (!obj_reg->is_humongous()) {
!         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
                 "Object end should be within the region");
        } else {
          size_t humongous_start = obj_reg->index();
!         size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
          for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
            check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
                   "Humongous object is in continuation that fits it");
          }
        }
--- 138,15 ---
        HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj);
        check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
               "Object start should be within the region");
  
        if (!obj_reg->is_humongous()) {
!         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->forward_safe_size()) <= obj_reg->top(),
                 "Object end should be within the region");
        } else {
          size_t humongous_start = obj_reg->index();
!         size_t humongous_end = humongous_start + (obj->forward_safe_size() >> ShenandoahHeapRegion::region_size_words_shift());
          for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
            check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
                   "Humongous object is in continuation that fits it");
          }
        }

*** 159,11 ***
        switch (_options._verify_liveness) {
          case ShenandoahVerifier::_verify_liveness_disable:
            // skip
            break;
          case ShenandoahVerifier::_verify_liveness_complete:
!           Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
            // fallthrough for fast failure for un-live regions:
          case ShenandoahVerifier::_verify_liveness_conservative:
            check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
                     "Object must belong to region with live data");
            break;
--- 159,11 ---
        switch (_options._verify_liveness) {
          case ShenandoahVerifier::_verify_liveness_disable:
            // skip
            break;
          case ShenandoahVerifier::_verify_liveness_complete:
!           Atomic::add(&_ld[obj_reg->index()], (uint) obj->forward_safe_size(), memory_order_relaxed);
            // fallthrough for fast failure for un-live regions:
          case ShenandoahVerifier::_verify_liveness_conservative:
            check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
                     "Object must belong to region with live data");
            break;

*** 200,11 ***
               "Should have no humongous forwardees");
  
        HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
        check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
               "Forwardee start should be within the region");
!       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
               "Forwardee end should be within the region");
  
        oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
        check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
               "Double forwarding");
--- 200,11 ---
               "Should have no humongous forwardees");
  
        HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
        check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
               "Forwardee start should be within the region");
!       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->forward_safe_size()) <= fwd_reg->top(),
               "Forwardee end should be within the region");
  
        oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
        check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
               "Double forwarding");

*** 303,11 ***
     * Verify oop fields from this object.
     * @param obj host object for verified fields
     */
    void verify_oops_from(oop obj) {
      _loc = obj;
!     obj->oop_iterate(this);
      _loc = nullptr;
    }
  
    virtual void do_oop(oop* p) { do_oop_work(p); }
    virtual void do_oop(narrowOop* p) { do_oop_work(p); }
--- 303,12 ---
     * Verify oop fields from this object.
     * @param obj host object for verified fields
     */
    void verify_oops_from(oop obj) {
      _loc = obj;
!     Klass* klass = obj->forward_safe_klass();
+     obj->oop_iterate_backwards(this, klass);
      _loc = nullptr;
    }
  
    virtual void do_oop(oop* p) { do_oop_work(p); }
    virtual void do_oop(narrowOop* p) { do_oop_work(p); }

*** 583,11 ***
      oop obj = cast_to_oop(addr);
      cl.verify_oop_standalone(obj);
  
      // Verify everything reachable from that object too, hopefully realizing
      // everything was already marked, and never touching further:
!     if (!is_instance_ref_klass(obj->klass())) {
        cl.verify_oops_from(obj);
        (*processed)++;
      }
      while (!stack.is_empty()) {
        ShenandoahVerifierTask task = stack.pop();
--- 584,11 ---
      oop obj = cast_to_oop(addr);
      cl.verify_oop_standalone(obj);
  
      // Verify everything reachable from that object too, hopefully realizing
      // everything was already marked, and never touching further:
!     if (!is_instance_ref_klass(obj->forward_safe_klass())) {
        cl.verify_oops_from(obj);
        (*processed)++;
      }
      while (!stack.is_empty()) {
        ShenandoahVerifierTask task = stack.pop();