src/share/vm/memory/threadLocalAllocBuffer.cpp

*** 51,60 ****
--- 51,62 ----
    for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
      thread->tlab().accumulate_statistics();
      thread->tlab().initialize_statistics();
    }
  
+   Universe::heap()->accumulate_statistics_all_gclabs();
+ 
    // Publish new stats if some allocation occurred.
    if (global_stats()->allocation() != 0) {
      global_stats()->publish();
      if (PrintTLAB) {
        global_stats()->print();
*** 66,76 ****
    Thread* thread = myThread();
    size_t capacity = Universe::heap()->tlab_capacity(thread);
    size_t used     = Universe::heap()->tlab_used(thread);
  
    _gc_waste += (unsigned)remaining();
!   size_t total_allocated = thread->allocated_bytes();
    size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc;
    _allocated_before_last_gc = total_allocated;
  
    if (PrintTLAB && (_number_of_refills > 0 || Verbose)) {
      print_stats("gc");
--- 68,78 ----
    Thread* thread = myThread();
    size_t capacity = Universe::heap()->tlab_capacity(thread);
    size_t used     = Universe::heap()->tlab_used(thread);
  
    _gc_waste += (unsigned)remaining();
!   size_t total_allocated = _gclab ? thread->allocated_bytes_gclab() : thread->allocated_bytes();
    size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc;
    _allocated_before_last_gc = total_allocated;
  
    if (PrintTLAB && (_number_of_refills > 0 || Verbose)) {
      print_stats("gc");
*** 114,124 ****
  void ThreadLocalAllocBuffer::make_parsable(bool retire) {
    if (end() != NULL) {
      invariants();
  
      if (retire) {
!       myThread()->incr_allocated_bytes(used_bytes());
      }
  
      CollectedHeap::fill_with_object(top(), hard_end(), retire);
  
      if (retire || ZeroTLAB) {  // "Reset" the TLAB
--- 116,130 ----
  void ThreadLocalAllocBuffer::make_parsable(bool retire) {
    if (end() != NULL) {
      invariants();
  
      if (retire) {
!       if (_gclab) {
!         myThread()->incr_allocated_bytes_gclab(used_bytes());
!       } else {
!         myThread()->incr_allocated_bytes(used_bytes());
!       }
      }
  
      CollectedHeap::fill_with_object(top(), hard_end(), retire);
  
      if (retire || ZeroTLAB) {  // "Reset" the TLAB
*** 192,202 ****
    set_pf_top(top);
    set_end(end);
    invariants();
  }
  
! void ThreadLocalAllocBuffer::initialize() {
    initialize(NULL,                    // start
               NULL,                    // top
               NULL);                   // end
  
    set_desired_size(initial_desired_size());
--- 198,210 ----
    set_pf_top(top);
    set_end(end);
    invariants();
  }
  
! void ThreadLocalAllocBuffer::initialize(bool gclab) {
!   _initialized = true;
!   _gclab = gclab;
    initialize(NULL,                    // start
               NULL,                    // top
               NULL);                   // end
  
    set_desired_size(initial_desired_size());
*** 226,236 ****
    _global_stats = new GlobalTLABStats();
  
    // During jvm startup, the main thread is initialized
    // before the heap is initialized.  So reinitialize it now.
    guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
!   Thread::current()->tlab().initialize();
  
    if (PrintTLAB && Verbose) {
      gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n",
                          min_size(), Thread::current()->tlab().initial_desired_size(), max_size());
    }
--- 234,247 ----
    _global_stats = new GlobalTLABStats();
  
    // During jvm startup, the main thread is initialized
    // before the heap is initialized.  So reinitialize it now.
    guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
!   Thread::current()->tlab().initialize(false);
!   if (UseShenandoahGC) {
!     Thread::current()->gclab().initialize(true);
!   }
  
    if (PrintTLAB && Verbose) {
      gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n",
                          min_size(), Thread::current()->tlab().initial_desired_size(), max_size());
    }
*** 258,273 ****
    size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste;
    size_t alloc = _number_of_refills * _desired_size;
    double waste_percent = alloc == 0 ? 0.0 :
                        100.0 * waste / alloc;
    size_t tlab_used  = Universe::heap()->tlab_used(thrd);
!   gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]"
                        " desired_size: " SIZE_FORMAT "KB"
                        " slow allocs: %d  refill waste: " SIZE_FORMAT "B"
                        " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
                        " slow: %dB fast: %dB\n",
!                       tag, thrd, thrd->osthread()->thread_id(),
                        _desired_size / (K / HeapWordSize),
                        _slow_allocations, _refill_waste_limit * HeapWordSize,
                        _allocation_fraction.average(),
                        _allocation_fraction.average() * tlab_used / K,
                        _number_of_refills, waste_percent,
--- 269,284 ----
    size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste;
    size_t alloc = _number_of_refills * _desired_size;
    double waste_percent = alloc == 0 ? 0.0 :
                        100.0 * waste / alloc;
    size_t tlab_used  = Universe::heap()->tlab_used(thrd);
!   gclog_or_tty->print("TLAB: %s %s thread: " INTPTR_FORMAT " [id: %2d]"
                        " desired_size: " SIZE_FORMAT "KB"
                        " slow allocs: %d  refill waste: " SIZE_FORMAT "B"
                        " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
                        " slow: %dB fast: %dB\n",
!                       tag, _gclab ? "gclab" : "tlab ", p2i(thrd), thrd->osthread()->thread_id(),
                        _desired_size / (K / HeapWordSize),
                        _slow_allocations, _refill_waste_limit * HeapWordSize,
                        _allocation_fraction.average(),
                        _allocation_fraction.average() * tlab_used / K,
                        _number_of_refills, waste_percent,
*** 287,299 ****
    }
    guarantee(p == top(), "end of last object must match end of space");
  }
  
  Thread* ThreadLocalAllocBuffer::myThread() {
!   return (Thread*)(((char *)this) +
!                    in_bytes(start_offset()) -
!                    in_bytes(Thread::tlab_start_offset()));
  }
  
  
  GlobalTLABStats::GlobalTLABStats() :
    _allocating_threads_avg(TLABAllocationWeight) {
--- 298,323 ----
    }
    guarantee(p == top(), "end of last object must match end of space");
  }
  
  Thread* ThreadLocalAllocBuffer::myThread() {
!   ByteSize gclab_offset = Thread::gclab_start_offset();
!   ByteSize tlab_offset = Thread::tlab_start_offset();
!   ByteSize offs = _gclab ? gclab_offset : tlab_offset;
!   Thread* thread = (Thread*)(((char *)this) +
!                    in_bytes(start_offset()) - in_bytes(offs));
! #ifdef ASSERT
!   assert(this == (_gclab ? &thread->gclab() : &thread->tlab()), "must be");
! #endif
!   return thread;
! }
! 
! void ThreadLocalAllocBuffer::rollback(size_t size) {
!   HeapWord* old_top = top();
!   if (old_top != NULL) { // Pathological case: we accept that we can't rollback.
!     set_top(old_top - size);
!   }
  }
  
  
  GlobalTLABStats::GlobalTLABStats() :
    _allocating_threads_avg(TLABAllocationWeight) {