8210883: ZGC: Parallel retire/resize/remap of TLABs

Reviewed-by: eosterlund

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -117,6 +117,10 @@ bool ZCollectedHeap::is_in_closed_subset(const void* p) const {
   return is_in(p);
 }
 
+void ZCollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
+  // Does nothing, not a parsable heap
+}
+
 HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) {
   const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(requested_size));
   const uintptr_t addr = _heap.alloc_tlab(size_in_bytes);

src/hotspot/share/gc/z/zCollectedHeap.hpp

@@ -56,9 +56,6 @@ private:
 public:
   static ZCollectedHeap* heap();
 
-  using CollectedHeap::ensure_parsability;
-  using CollectedHeap::resize_all_tlabs;
-
   ZCollectedHeap(ZCollectorPolicy* policy);
   virtual Name kind() const;
   virtual const char* name() const;
@@ -78,6 +75,8 @@ public:
   virtual bool is_in(const void* p) const;
   virtual bool is_in_closed_subset(const void* p) const;
 
+  virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
+
   virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded);
   virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
                                                        size_t size,

src/hotspot/share/gc/z/zHeap.cpp

@@ -273,13 +273,13 @@ void ZHeap::mark_start() {
   // Update statistics
   ZStatSample(ZSamplerHeapUsedBeforeMark, used());
 
-  // Retire TLABs
-  _object_allocator.retire_tlabs();
-
   // Flip address view
   ZAddressMasks::flip_to_marked();
   flip_views();
 
+  // Retire allocating pages
+  _object_allocator.retire_pages();
+
   // Reset allocated/reclaimed/used statistics
   _page_allocator.reset_statistics();
 
@@ -475,9 +475,6 @@ void ZHeap::relocate_start() {
   ZAddressMasks::flip_to_remapped();
   flip_views();
 
-  // Remap TLABs
-  _object_allocator.remap_tlabs();
-
   // Enter relocate phase
   ZGlobalPhase = ZPhaseRelocate;
 

src/hotspot/share/gc/z/zInitialize.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,7 @@
 #include "gc/z/zLargePages.hpp"
 #include "gc/z/zNUMA.hpp"
 #include "gc/z/zStat.hpp"
+#include "gc/z/zStatTLAB.hpp"
 #include "gc/z/zTracer.hpp"
 #include "logging/log.hpp"
 #include "runtime/vm_version.hpp"
@@ -45,6 +46,7 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
   ZNUMA::initialize();
   ZCPU::initialize();
   ZStatValue::initialize();
+  ZStatTLAB::initialize();
   ZTracer::initialize();
   ZLargePages::initialize();
   ZBarrierSet::set_barrier_set(barrier_set);

src/hotspot/share/gc/z/zMark.cpp

@@ -32,6 +32,7 @@
 #include "gc/z/zPageTable.inline.hpp"
 #include "gc/z/zRootsIterator.hpp"
 #include "gc/z/zStat.hpp"
+#include "gc/z/zStatTLAB.hpp"
 #include "gc/z/zTask.hpp"
 #include "gc/z/zThread.hpp"
 #include "gc/z/zUtils.inline.hpp"
@@ -119,11 +120,25 @@ void ZMark::prepare_mark() {
 
 class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure {
 public:
+  ZMarkRootsIteratorClosure() {
+    ZStatTLAB::reset();
+  }
+
+  ~ZMarkRootsIteratorClosure() {
+    ZStatTLAB::publish();
+  }
+
   virtual void do_thread(Thread* thread) {
     ZRootsIteratorClosure::do_thread(thread);
 
     // Update thread local address bad mask
     ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
+
+    // Retire TLAB
+    if (UseTLAB && thread->is_Java_thread()) {
+      thread->tlab().retire(ZStatTLAB::get());
+      thread->tlab().resize();
+    }
   }
 
   virtual void do_oop(oop* p) {
@@ -137,8 +152,9 @@ public:
 
 class ZMarkRootsTask : public ZTask {
 private:
-  ZMark* const   _mark;
-  ZRootsIterator _roots;
+  ZMark* const              _mark;
+  ZRootsIterator            _roots;
+  ZMarkRootsIteratorClosure _cl;
 
 public:
   ZMarkRootsTask(ZMark* mark) :
@@ -147,8 +163,7 @@ public:
       _roots() {}
 
   virtual void work() {
-    ZMarkRootsIteratorClosure cl;
-    _roots.oops_do(&cl);
+    _roots.oops_do(&_cl);
 
     // Flush and free worker stacks. Needed here since
     // the set of workers executing during root scanning

src/hotspot/share/gc/z/zObjectAllocator.cpp

@@ -22,7 +22,6 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 #include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
@@ -41,8 +40,6 @@
 
 static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);
-static const ZStatSubPhase ZSubPhasePauseRetireTLABS("Pause Retire TLABS");
-static const ZStatSubPhase ZSubPhasePauseRemapTLABS("Pause Remap TLABS");
 
 ZObjectAllocator::ZObjectAllocator(uint nworkers) :
     _nworkers(nworkers),
@@ -293,17 +290,9 @@ size_t ZObjectAllocator::remaining() const {
   return 0;
 }
 
-void ZObjectAllocator::retire_tlabs() {
-  ZStatTimer timer(ZSubPhasePauseRetireTLABS);
+void ZObjectAllocator::retire_pages() {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
-  // Retire TLABs
-  if (UseTLAB) {
-    ZCollectedHeap* heap = ZCollectedHeap::heap();
-    heap->ensure_parsability(true /* retire_tlabs */);
-    heap->resize_all_tlabs();
-  }
-
   // Reset used
   _used.set_all(0);
 
@@ -312,18 +301,3 @@ void ZObjectAllocator::retire_tlabs() {
   _shared_small_page.set_all(NULL);
   _worker_small_page.set_all(NULL);
 }
-
-static void remap_tlab_address(HeapWord** p) {
-  *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p);
-}
-
-void ZObjectAllocator::remap_tlabs() {
-  ZStatTimer timer(ZSubPhasePauseRemapTLABS);
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-
-  if (UseTLAB) {
-    for (JavaThreadIteratorWithHandle iter; JavaThread* thread = iter.next(); ) {
-      thread->tlab().addresses_do(remap_tlab_address);
-    }
-  }
-}

src/hotspot/share/gc/z/zObjectAllocator.hpp

@@ -72,8 +72,7 @@ public:
   size_t used() const;
   size_t remaining() const;
 
-  void retire_tlabs();
-  void remap_tlabs();
+  void retire_pages();
 };
 
 #endif // SHARE_GC_Z_ZOBJECTALLOCATOR_HPP

src/hotspot/share/gc/z/zRelocate.cpp

@@ -22,6 +22,8 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zHeap.hpp"
 #include "gc/z/zOopClosures.inline.hpp"
 #include "gc/z/zPage.hpp"
@@ -35,12 +37,22 @@ ZRelocate::ZRelocate(ZWorkers* workers) :
     _workers(workers) {}
 
 class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure {
+private:
+  static void remap_address(HeapWord** p) {
+    *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p);
+  }
+
 public:
   virtual void do_thread(Thread* thread) {
     ZRootsIteratorClosure::do_thread(thread);
 
     // Update thread local address bad mask
     ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
+
+    // Remap TLAB
+    if (UseTLAB && thread->is_Java_thread()) {
+      thread->tlab().addresses_do(remap_address);
+    }
   }
 
   virtual void do_oop(oop* p) {
@@ -54,7 +66,8 @@ public:
 
 class ZRelocateRootsTask : public ZTask {
 private:
-  ZRootsIterator _roots;
+  ZRootsIterator                _roots;
+  ZRelocateRootsIteratorClosure _cl;
 
 public:
   ZRelocateRootsTask() :
@@ -64,8 +77,7 @@ public:
   virtual void work() {
     // During relocation we need to visit the JVMTI
     // export weak roots to rehash the JVMTI tag map
-    ZRelocateRootsIteratorClosure cl;
-    _roots.oops_do(&cl, true /* visit_jvmti_weak_export */);
+    _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
   }
 };
 

src/hotspot/share/gc/z/zStatTLAB.cpp

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "gc/z/zStatTLAB.hpp"
+
+ZPerWorker<ThreadLocalAllocStats>* ZStatTLAB::_stats = NULL;
+
+void ZStatTLAB::initialize() {
+  if (UseTLAB) {
+    assert(_stats == NULL, "Already initialized");
+    _stats = new ZPerWorker<ThreadLocalAllocStats>();
+    reset();
+  }
+}
+
+void ZStatTLAB::reset() {
+  if (UseTLAB) {
+    ZPerWorkerIterator<ThreadLocalAllocStats> iter(_stats);
+    for (ThreadLocalAllocStats* stats; iter.next(&stats);) {
+      stats->reset();
+    }
+  }
+}
+
+ThreadLocalAllocStats* ZStatTLAB::get() {
+  if (UseTLAB) {
+    return _stats->addr();
+  }
+
+  return NULL;
+}
+
+void ZStatTLAB::publish() {
+  if (UseTLAB) {
+    ThreadLocalAllocStats total;
+
+    ZPerWorkerIterator<ThreadLocalAllocStats> iter(_stats);
+    for (ThreadLocalAllocStats* stats; iter.next(&stats);) {
+      total.update(*stats);
+    }
+
+    total.publish();
+  }
+}

src/hotspot/share/gc/z/zStatTLAB.hpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZSTATTLAB_HPP
+#define SHARE_GC_Z_ZSTATTLAB_HPP
+
+#include "gc/shared/threadLocalAllocBuffer.hpp"
+#include "gc/z/zValue.hpp"
+#include "memory/allocation.hpp"
+
+class ZStatTLAB : public AllStatic {
+private:
+  static ZPerWorker<ThreadLocalAllocStats>* _stats;
+
+public:
+  static void initialize();
+  static void reset();
+  static ThreadLocalAllocStats* get();
+  static void publish();
+};
+
+#endif // SHARE_GC_Z_ZSTATTLAB_HPP

src/hotspot/share/gc/z/zValue.hpp

@@ -131,7 +131,7 @@ template <typename S, typename T>
 class ZValueIterator;
 
 template <typename S, typename T>
-class ZValue {
+class ZValue : public CHeapObj<mtGC> {
 private:
   const uintptr_t _addr;