src/share/vm/memory/threadLocalAllocBuffer.cpp

@@ -51,10 +51,12 @@
   for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
     thread->tlab().accumulate_statistics();
     thread->tlab().initialize_statistics();
   }
 
+  Universe::heap()->accumulate_statistics_all_gclabs();
+
   // Publish new stats if some allocation occurred.
   if (global_stats()->allocation() != 0) {
     global_stats()->publish();
     if (PrintTLAB) {
       global_stats()->print();

@@ -66,11 +68,11 @@
   Thread* thread = myThread();
   size_t capacity = Universe::heap()->tlab_capacity(thread);
   size_t used     = Universe::heap()->tlab_used(thread);
 
   _gc_waste += (unsigned)remaining();
-  size_t total_allocated = thread->allocated_bytes();
+  size_t total_allocated = _gclab ? thread->allocated_bytes_gclab() : thread->allocated_bytes();
   size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc;
   _allocated_before_last_gc = total_allocated;
 
   if (PrintTLAB && (_number_of_refills > 0 || Verbose)) {
     print_stats("gc");

@@ -114,11 +116,15 @@
 void ThreadLocalAllocBuffer::make_parsable(bool retire) {
   if (end() != NULL) {
     invariants();
 
     if (retire) {
-      myThread()->incr_allocated_bytes(used_bytes());
+      if (_gclab) {
+        myThread()->incr_allocated_bytes_gclab(used_bytes());
+      } else {
+        myThread()->incr_allocated_bytes(used_bytes());
+      }
     }
 
     CollectedHeap::fill_with_object(top(), hard_end(), retire);
 
     if (retire || ZeroTLAB) {  // "Reset" the TLAB

@@ -192,11 +198,13 @@
   set_pf_top(top);
   set_end(end);
   invariants();
 }
 
-void ThreadLocalAllocBuffer::initialize() {
+void ThreadLocalAllocBuffer::initialize(bool gclab) {
+  _initialized = true;
+  _gclab = gclab;
   initialize(NULL,                    // start
              NULL,                    // top
              NULL);                   // end
 
   set_desired_size(initial_desired_size());

@@ -226,11 +234,14 @@
   _global_stats = new GlobalTLABStats();
 
   // During jvm startup, the main thread is initialized
   // before the heap is initialized.  So reinitialize it now.
   guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
-  Thread::current()->tlab().initialize();
+  Thread::current()->tlab().initialize(false);
+  if (UseShenandoahGC) {
+    Thread::current()->gclab().initialize(true);
+  }
 
   if (PrintTLAB && Verbose) {
     gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n",
                         min_size(), Thread::current()->tlab().initial_desired_size(), max_size());
   }

@@ -258,16 +269,16 @@
   size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste;
   size_t alloc = _number_of_refills * _desired_size;
   double waste_percent = alloc == 0 ? 0.0 :
                       100.0 * waste / alloc;
   size_t tlab_used  = Universe::heap()->tlab_used(thrd);
-  gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]"
+  gclog_or_tty->print("TLAB: %s %s thread: " INTPTR_FORMAT " [id: %2d]"
                       " desired_size: " SIZE_FORMAT "KB"
                       " slow allocs: %d  refill waste: " SIZE_FORMAT "B"
                       " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
                       " slow: %dB fast: %dB\n",
-                      tag, thrd, thrd->osthread()->thread_id(),
+                      tag, _gclab ? "gclab" : "tlab ", p2i(thrd), thrd->osthread()->thread_id(),
                       _desired_size / (K / HeapWordSize),
                       _slow_allocations, _refill_waste_limit * HeapWordSize,
                       _allocation_fraction.average(),
                       _allocation_fraction.average() * tlab_used / K,
                       _number_of_refills, waste_percent,

@@ -287,13 +298,26 @@
   }
   guarantee(p == top(), "end of last object must match end of space");
 }
 
 Thread* ThreadLocalAllocBuffer::myThread() {
-  return (Thread*)(((char *)this) +
-                   in_bytes(start_offset()) -
-                   in_bytes(Thread::tlab_start_offset()));
+  ByteSize gclab_offset = Thread::gclab_start_offset();
+  ByteSize tlab_offset = Thread::tlab_start_offset();
+  ByteSize offs = _gclab ? gclab_offset : tlab_offset;
+  Thread* thread = (Thread*)(((char *)this) +
+                   in_bytes(start_offset()) - in_bytes(offs));
+#ifdef ASSERT
+  assert(this == (_gclab ? &thread->gclab() : &thread->tlab()), "must be");
+#endif
+  return thread;
+}
+
+void ThreadLocalAllocBuffer::rollback(size_t size) {
+  HeapWord* old_top = top();
+  if (old_top != NULL) { // Pathological case: we accept that we can't rollback.
+    set_top(old_top - size);
+  }
 }
 
 
 GlobalTLABStats::GlobalTLABStats() :
   _allocating_threads_avg(TLABAllocationWeight) {