8291037: Move PLAB resizing mechanism to G1EvacStats

Reviewed-by: sangheki, iwalulya

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -2838,8 +2838,8 @@ G1JFRTracerMark::~G1JFRTracerMark() {
 void G1CollectedHeap::prepare_tlabs_for_mutator() {
   Ticks start = Ticks::now();
 
-  _survivor_evac_stats.adjust_desired_plab_sz();
-  _old_evac_stats.adjust_desired_plab_sz();
+  _survivor_evac_stats.adjust_desired_plab_size();
+  _old_evac_stats.adjust_desired_plab_size();
 
   allocate_dummy_regions();
 

src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp

@@ -66,7 +66,7 @@ G1EvacStats* G1CollectedHeap::alloc_buffer_stats(G1HeapRegionAttr dest) {
 }
 
 size_t G1CollectedHeap::desired_plab_sz(G1HeapRegionAttr dest) {
-  size_t gclab_word_size = alloc_buffer_stats(dest)->desired_plab_sz(workers()->active_workers());
+  size_t gclab_word_size = alloc_buffer_stats(dest)->desired_plab_size(workers()->active_workers());
   // Prevent humongous PLAB sizes for two reasons:
   // * PLABs are allocated using a similar paths as oops, but should
   //   never be in a humongous region

src/hotspot/share/gc/g1/g1EvacStats.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,18 @@
 #include "runtime/globals.hpp"
 
 void G1EvacStats::log_plab_allocation() {
-  PLABStats::log_plab_allocation();
+  log_debug(gc, plab)("%s PLAB allocation: "
+                      "allocated: %zuB, "
+                      "wasted: %zuB, "
+                      "unused: %zuB, "
+                      "used: %zuB, "
+                      "undo waste: %zuB, ",
+                      _description,
+                      _allocated * HeapWordSize,
+                      _wasted * HeapWordSize,
+                      _unused * HeapWordSize,
+                      used() * HeapWordSize,
+                      _undo_wasted * HeapWordSize);
   log_debug(gc, plab)("%s other allocation: "
                       "region end waste: %zuB, "
                       "regions filled: %u, "
@@ -50,7 +61,16 @@ void G1EvacStats::log_plab_allocation() {
                       _failure_waste * HeapWordSize);
 }
 
-size_t G1EvacStats::compute_desired_plab_sz() {
+void G1EvacStats::log_sizing(size_t calculated_words, size_t net_desired_words) {
+  log_debug(gc, plab)("%s sizing: "
+                      "calculated: %zuB, "
+                      "actual: %zuB",
+                      _description,
+                      calculated_words * HeapWordSize,
+                      net_desired_words * HeapWordSize);
+}
+
+size_t G1EvacStats::compute_desired_plab_size() const {
   // The size of the PLAB caps the amount of space that can be wasted at the
   // end of the collection. In the worst case the last PLAB could be completely
   // empty.
@@ -90,12 +110,14 @@ size_t G1EvacStats::compute_desired_plab_sz() {
   size_t const used_for_waste_calculation = used() > _region_end_waste ? used() - _region_end_waste : 0;
 
   size_t const total_waste_allowed = used_for_waste_calculation * TargetPLABWastePct;
-  size_t const cur_plab_sz = (size_t)((double)total_waste_allowed / G1LastPLABAverageOccupancy);
-  return cur_plab_sz;
+  return (size_t)((double)total_waste_allowed / G1LastPLABAverageOccupancy);
 }
 
 G1EvacStats::G1EvacStats(const char* description, size_t default_per_thread_plab_size, unsigned wt) :
-  PLABStats(description, default_per_thread_plab_size, default_per_thread_plab_size * ParallelGCThreads, wt),
+  PLABStats(description),
+  _default_plab_size(default_per_thread_plab_size),
+  _desired_net_plab_size(default_per_thread_plab_size * ParallelGCThreads),
+  _net_plab_size_filter(wt),
   _region_end_waste(0),
   _regions_filled(0),
   _num_plab_filled(0),
@@ -105,5 +127,36 @@ G1EvacStats::G1EvacStats(const char* description, size_t default_per_thread_plab
   _failure_waste(0) {
 }
 
+// Calculates plab size for current number of gc worker threads.
+size_t G1EvacStats::desired_plab_size(uint no_of_gc_workers) const {
+  if (!ResizePLAB) {
+      return _default_plab_size;
+  }
+  return align_object_size(clamp(_desired_net_plab_size / no_of_gc_workers, min_size(), max_size()));
+}
 
-G1EvacStats::~G1EvacStats() { }
+void G1EvacStats::adjust_desired_plab_size() {
+  log_plab_allocation();
+
+  if (ResizePLAB) {
+    assert(is_object_aligned(max_size()) && min_size() <= max_size(),
+           "PLAB clipping computation may be incorrect");
+
+    assert(_allocated != 0 || _unused == 0,
+           "Inconsistency in PLAB stats: "
+           "_allocated: %zu, "
+           "_wasted: %zu, "
+           "_unused: %zu, "
+           "_undo_wasted: %zu",
+           _allocated, _wasted, _unused, _undo_wasted);
+
+    size_t plab_size = compute_desired_plab_size();
+    // Take historical weighted average
+    _net_plab_size_filter.sample(plab_size);
+    _desired_net_plab_size = MAX2(min_size(), (size_t)_net_plab_size_filter.average());
+
+    log_sizing(plab_size, _desired_net_plab_size);
+  }
+  // Clear accumulators for next round
+  reset();
+}

src/hotspot/share/gc/g1/g1EvacStats.hpp

@@ -26,10 +26,16 @@
 #define SHARE_GC_G1_G1EVACSTATS_HPP
 
 #include "gc/shared/plab.hpp"
+#include "gc/shared/gcUtil.hpp"
 
-// Records various memory allocation statistics gathered during evacuation.
+// Records various memory allocation statistics gathered during evacuation. All sizes
+// are in HeapWords.
 class G1EvacStats : public PLABStats {
- private:
+  size_t _default_plab_size;
+  size_t _desired_net_plab_size; // Output of filter (below), suitably trimmed and quantized
+  AdaptiveWeightedAverage
+         _net_plab_size_filter;  // Integrator with decay
+
   size_t _region_end_waste; // Number of words wasted due to skipping to the next region.
   uint   _regions_filled;   // Number of regions filled completely.
   size_t _num_plab_filled; // Number of PLABs filled and retired.
@@ -55,14 +61,21 @@ class G1EvacStats : public PLABStats {
     _failure_waste = 0;
   }
 
-  virtual void log_plab_allocation();
+  void log_plab_allocation();
+  void log_sizing(size_t calculated_words, size_t net_desired_words);
 
-  virtual size_t compute_desired_plab_sz();
+  size_t compute_desired_plab_size() const;
 
- public:
+public:
   G1EvacStats(const char* description, size_t default_per_thread_plab_size, unsigned wt);
 
-  ~G1EvacStats();
+  // Calculates plab size for current number of gc worker threads.
+  size_t desired_plab_size(uint no_of_gc_workers) const;
+
+  // Computes the new desired PLAB size assuming one gc worker thread, updating
+  // _desired_plab_sz, and clearing statistics for the next GC.
+  // Should be called at the end of a GC pause.
+  void adjust_desired_plab_size();
 
   uint regions_filled() const { return _regions_filled; }
   size_t num_plab_filled() const { return _num_plab_filled; }

src/hotspot/share/gc/shared/plab.cpp

@@ -114,81 +114,3 @@ void PLAB::undo_allocation(HeapWord* obj, size_t word_sz) {
     add_undo_waste(obj, word_sz);
   }
 }
-
-void PLABStats::log_plab_allocation() {
-  log_debug(gc, plab)("%s PLAB allocation: "
-                      "allocated: " SIZE_FORMAT "B, "
-                      "wasted: " SIZE_FORMAT "B, "
-                      "unused: " SIZE_FORMAT "B, "
-                      "used: " SIZE_FORMAT "B, "
-                      "undo waste: " SIZE_FORMAT "B, ",
-                      _description,
-                      _allocated * HeapWordSize,
-                      _wasted * HeapWordSize,
-                      _unused * HeapWordSize,
-                      used() * HeapWordSize,
-                      _undo_wasted * HeapWordSize);
-}
-
-void PLABStats::log_sizing(size_t calculated_words, size_t net_desired_words) {
-  log_debug(gc, plab)("%s sizing: "
-                      "calculated: " SIZE_FORMAT "B, "
-                      "actual: " SIZE_FORMAT "B",
-                      _description,
-                      calculated_words * HeapWordSize,
-                      net_desired_words * HeapWordSize);
-}
-
-// Calculates plab size for current number of gc worker threads.
-size_t PLABStats::desired_plab_sz(uint no_of_gc_workers) {
-  if (!ResizePLAB) {
-      return _default_plab_sz;
-  }
-  return align_object_size(clamp(_desired_net_plab_sz / no_of_gc_workers, min_size(), max_size()));
-}
-
-// Compute desired plab size for one gc worker thread and latch result for later
-// use. This should be called once at the end of parallel
-// scavenge; it clears the sensor accumulators.
-void PLABStats::adjust_desired_plab_sz() {
-  log_plab_allocation();
-
-  if (!ResizePLAB) {
-    // Clear accumulators for next round.
-    reset();
-    return;
-  }
-
-  assert(is_object_aligned(max_size()) && min_size() <= max_size(),
-         "PLAB clipping computation may be incorrect");
-
-  assert(_allocated != 0 || _unused == 0,
-         "Inconsistency in PLAB stats: "
-         "_allocated: " SIZE_FORMAT ", "
-         "_wasted: " SIZE_FORMAT ", "
-         "_unused: " SIZE_FORMAT ", "
-         "_undo_wasted: " SIZE_FORMAT,
-         _allocated, _wasted, _unused, _undo_wasted);
-
-  size_t plab_sz = compute_desired_plab_sz();
-  // Take historical weighted average
-  _filter.sample(plab_sz);
-  _desired_net_plab_sz = MAX2(min_size(), (size_t)_filter.average());
-
-  log_sizing(plab_sz, _desired_net_plab_sz);
-  // Clear accumulators for next round
-  reset();
-}
-
-size_t PLABStats::compute_desired_plab_sz() {
-  size_t allocated      = MAX2(_allocated, size_t(1));
-  double wasted_frac    = (double)_unused / (double)allocated;
-  size_t target_refills = (size_t)((wasted_frac * TargetSurvivorRatio) / TargetPLABWastePct);
-  if (target_refills == 0) {
-    target_refills = 1;
-  }
-  size_t used = allocated - _wasted - _unused;
-  // Assumed to have 1 gc worker thread
-  size_t recent_plab_sz = used / target_refills;
-  return recent_plab_sz;
-}

src/hotspot/share/gc/shared/plab.hpp

@@ -26,7 +26,6 @@
 #define SHARE_GC_SHARED_PLAB_HPP
 
 #include "gc/shared/collectedHeap.hpp"
-#include "gc/shared/gcUtil.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/globalDefinitions.hpp"
 
@@ -143,17 +142,13 @@ public:
 
 // PLAB book-keeping.
 class PLABStats : public CHeapObj<mtGC> {
- protected:
+protected:
   const char* _description;   // Identifying string.
 
   size_t _allocated;          // Total allocated
   size_t _wasted;             // of which wasted (internal fragmentation)
   size_t _undo_wasted;        // of which wasted on undo (is not used for calculation of PLAB size)
   size_t _unused;             // Unused in last buffer
-  size_t _default_plab_sz;
-  size_t _desired_net_plab_sz;// Output of filter (below), suitably trimmed and quantized
-  AdaptiveWeightedAverage
-         _filter;             // Integrator with decay
 
   virtual void reset() {
     _allocated   = 0;
@@ -162,22 +157,13 @@ class PLABStats : public CHeapObj<mtGC> {
     _unused      = 0;
   }
 
-  virtual void log_plab_allocation();
-  virtual void log_sizing(size_t calculated, size_t net_desired);
-
-  // helper for adjust_desired_plab_sz().
-  virtual size_t compute_desired_plab_sz();
-
- public:
-  PLABStats(const char* description, size_t default_per_thread_plab_size, size_t desired_net_plab_sz, unsigned wt) :
+public:
+  PLABStats(const char* description) :
     _description(description),
     _allocated(0),
     _wasted(0),
     _undo_wasted(0),
-    _unused(0),
-    _default_plab_sz(default_per_thread_plab_size),
-    _desired_net_plab_sz(desired_net_plab_sz),
-    _filter(wt)
+    _unused(0)
   { }
 
   virtual ~PLABStats() { }
@@ -196,13 +182,6 @@ class PLABStats : public CHeapObj<mtGC> {
     return PLAB::max_size();
   }
 
-  // Calculates plab size for current number of gc worker threads.
-  size_t desired_plab_sz(uint no_of_gc_workers);
-
-  // Updates the current desired PLAB size. Computes the new desired PLAB size with one gc worker thread,
-  // updates _desired_plab_sz and clears sensor accumulators.
-  void adjust_desired_plab_sz();
-
   inline void add_allocated(size_t v);
 
   inline void add_unused(size_t v);