src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -59,11 +59,11 @@
    Prefetch::write(obj->mark_addr(), 0);
    push_depth(ScannerTask(p));
  }
  
  inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
-                                                       size_t obj_size,
+                                                       Klass* klass, size_t obj_size,
                                                        uint age, bool tenured,
                                                        const PSPromotionLAB* lab) {
    // Skip if memory allocation failed
    if (new_obj != NULL) {
      const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer();

@@ -71,18 +71,18 @@
      if (lab != NULL) {
        // Promotion of object through newly allocated PLAB
        if (gc_tracer->should_report_promotion_in_new_plab_event()) {
          size_t obj_bytes = obj_size * HeapWordSize;
          size_t lab_size = lab->capacity();
-         gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes,
+         gc_tracer->report_promotion_in_new_plab_event(klass, obj_bytes,
                                                        age, tenured, lab_size);
        }
      } else {
        // Promotion of object directly to heap
        if (gc_tracer->should_report_promotion_outside_plab_event()) {
          size_t obj_bytes = obj_size * HeapWordSize;
-         gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes,
+         gc_tracer->report_promotion_outside_plab_event(klass, obj_bytes,
                                                         age, tenured);
        }
      }
    }
  }

@@ -142,11 +142,11 @@
      // Ensure any loads from the forwardee follow all changes that precede
      // the release-cmpxchg that performed the forwarding, possibly in some
      // other thread.
      OrderAccess::acquire();
      // Return the already installed forwardee.
-     return cast_to_oop(m.decode_pointer());
+     return o->forwardee(m);
    }
  }
  
  //
  // This method is pretty bulky. It would be nice to split it up

@@ -158,11 +158,18 @@
                                                                 markWord test_mark) {
    assert(should_scavenge(&o), "Sanity");
  
    oop new_obj = NULL;
    bool new_obj_is_tenured = false;
-   size_t new_obj_size = o->size();
+   // NOTE: With compact headers, it is not safe to load the Klass* from o, because
+   // that would access the mark-word, and the mark-word might change at any time by
+   // concurrent promotion. The promoted mark-word would point to the forwardee, which
+   // may not yet have completed copying. Therefore we must load the Klass* from
+   // the mark-word that we have already loaded. This is safe, because we have checked
+   // that this is not yet forwarded in the caller.
+   Klass* klass = o->forward_safe_klass(test_mark);
+   size_t new_obj_size = o->size_given_klass(klass);
  
    // Find the objects age, MT safe.
    uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
        test_mark.displaced_mark_helper().age() : test_mark.age();
  

@@ -173,21 +180,21 @@
        if (new_obj == NULL && !_young_gen_is_full) {
          // Do we allocate directly, or flush and refill?
          if (new_obj_size > (YoungPLABSize / 2)) {
            // Allocate this object directly
            new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size));
-           promotion_trace_event(new_obj, o, new_obj_size, age, false, NULL);
+           promotion_trace_event(new_obj, o, klass, new_obj_size, age, false, NULL);
          } else {
            // Flush and fill
            _young_lab.flush();
  
            HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
            if (lab_base != NULL) {
              _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
              // Try the young lab allocation again.
              new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
-             promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab);
+             promotion_trace_event(new_obj, o, klass, new_obj_size, age, false, &_young_lab);
            } else {
              _young_gen_is_full = true;
            }
          }
        }

@@ -209,11 +216,11 @@
        if (!_old_gen_is_full) {
          // Do we allocate directly, or flush and refill?
          if (new_obj_size > (OldPLABSize / 2)) {
            // Allocate this object directly
            new_obj = cast_to_oop(old_gen()->allocate(new_obj_size));
-           promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL);
+           promotion_trace_event(new_obj, o, klass, new_obj_size, age, true, NULL);
          } else {
            // Flush and fill
            _old_lab.flush();
  
            HeapWord* lab_base = old_gen()->allocate(OldPLABSize);

@@ -226,11 +233,11 @@
              }
  #endif
              _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
              // Try the old lab allocation again.
              new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
-             promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab);
+             promotion_trace_event(new_obj, o, klass, new_obj_size, age, true, &_old_lab);
            }
          }
        }
  
        // This is the promotion failed test, and code handling.