Add option to aggregate pool stats to reduce ODS counter inflation

Summary:
Implements configurable aggregated pool stats for cachelib to reduce ODS counter inflation. Adds aggregatePoolStats config option that combines all pool statistics into a single "aggregated" entry before export. Maintains backward compatibility (disabled by default) and includes comprehensive test coverage.

Key changes:
- Added CacheAllocatorConfig::enableAggregatePoolStats() configuration
- Implemented updateAggregatedPoolStats()` method using PoolStats::operator+=
- Adapted PoolStats::operator+= to allow aggregation of PoolStats with different allocation classes
- Modified exportStats() to conditionally aggregate based on config
- Enhanced PoolStats aggregation for pool-level metrics

Reviewed By: rlyerly, pbhandar2

Differential Revision: D79768148

fbshipit-source-id: 29a9fb80b5c946be7da0d9b22ec3ee8b629380b3

Author: AlnisM

cachelib/allocator/Cache.cpp

@@ -16,6 +16,8 @@
 
 #include "cachelib/allocator/Cache.h"
 
+#include <folly/logging/xlog.h>
+
 #include <mutex>
 
 #include "cachelib/allocator/RebalanceStrategy.h"
@@ -89,6 +91,11 @@ void CacheBase::updateObjectCacheStats(const std::string& statPrefix) const {
 void CacheBase::updatePoolStats(const std::string& statPrefix,
                                 PoolId pid) const {
   const PoolStats stats = getPoolStats(pid);
+  updatePoolStats(statPrefix, stats);
+}
+
+void CacheBase::updatePoolStats(const std::string& statPrefix,
+                                const PoolStats& stats) const {
   const std::string prefix = statPrefix + "pool." + stats.poolName + ".";
 
   counters_.updateCount(prefix + "size", stats.poolSize);
@@ -531,6 +538,12 @@ CacheBase::CacheHitRate CacheBase::calculateCacheHitRate(
   return {overall, ram, nvm};
 }
 
+void CacheBase::updateIndividualPoolStats(const std::string& statPrefix) const {
+  for (const auto pid : getRegularPoolIds()) {
+    updatePoolStats(statPrefix, pid);
+  }
+}
+
 void CacheBase::exportStats(
     const std::string& statPrefix,
     std::chrono::seconds aggregationInterval,
@@ -539,8 +552,15 @@ void CacheBase::exportStats(
   updateNvmCacheStats(statPrefix);
   updateEventTrackerStats(statPrefix);
 
-  for (const auto pid : getRegularPoolIds()) {
-    updatePoolStats(statPrefix, pid);
+  if (aggregatePoolStats_ && canAggregatePoolStats()) {
+    updateAggregatedPoolStats(statPrefix);
+  } else {
+    // Log warning when aggregation is enabled but not possible
+    if (aggregatePoolStats_) {
+      XLOG(WARN) << "Pool stats aggregation is enabled but cannot be performed "
+                    "due to too many allocation classes";
+    }
+    updateIndividualPoolStats(statPrefix);
   }
 
   for (const auto pid : getCCachePoolIds()) {
@@ -556,4 +576,37 @@ void CacheBase::exportStats(
 
   return counters_.exportStats(aggregationInterval, cb);
 }
+
+bool CacheBase::canAggregatePoolStats() const {
+  const auto poolIds = getRegularPoolIds();
+  XDCHECK(!poolIds.empty(), "Regular pool IDs should not be empty");
+
+  // Collect all unique allocation sizes from all pools
+  std::unordered_set<uint32_t> allAllocSizes;
+  for (const auto pid : poolIds) {
+    const auto& pool = getPool(pid);
+    for (const auto& allocSize : pool.getAllocSizes()) {
+      allAllocSizes.insert(allocSize);
+    }
+  }
+  return allAllocSizes.size() <= MemoryAllocator::kMaxClasses;
+}
+
+void CacheBase::updateAggregatedPoolStats(const std::string& statPrefix) const {
+  const auto poolIds = getRegularPoolIds();
+  XDCHECK(!poolIds.empty(), "Regular pool IDs should not be empty");
+  // Get the first pool stats to initialize the aggregated stats
+  auto poolIdsIter = poolIds.begin();
+  PoolStats aggregatedStats = getPoolStats(*poolIdsIter);
+  ++poolIdsIter;
+
+  // Aggregate all remaining pool stats
+  for (; poolIdsIter != poolIds.end(); ++poolIdsIter) {
+    const PoolStats stats = getPoolStats(*poolIdsIter);
+    aggregatedStats += stats;
+  }
+  aggregatedStats.poolName = "aggregated";
+  updatePoolStats(statPrefix, aggregatedStats);
+}
+
 } // namespace facebook::cachelib

cachelib/allocator/Cache.h

@@ -216,6 +216,9 @@ class CacheBase {
   // <Stat -> Count/Delta> maps
   mutable RateMap counters_;
 
+  // Whether to aggregate pool stats to reduce ODS counter inflation
+  bool aggregatePoolStats_{false};
+
  protected:
   // move bytes from one pool to another. The source pool should be at least
   // _bytes_ in size.
@@ -304,6 +307,21 @@ class CacheBase {
   //   @param pid    the poolId that needs updating
   void updatePoolStats(const std::string& statPrefix, PoolId pid) const;
 
+  // Update pool stats with a PoolStats object directly
+  //   @param stats  the PoolStats object to update
+  void updatePoolStats(const std::string& statPrefix,
+                       const PoolStats& stats) const;
+
+  // Update individual pool stats (each pool reported separately)
+  void updateIndividualPoolStats(const std::string& statPrefix) const;
+
+  // Update aggregated pool stats (all pools combined into one stat)
+  void updateAggregatedPoolStats(const std::string& statPrefix) const;
+
+  // Returns true if the number of distinct allocation sizes across all pools is
+  // less than the maximum number of allocation sizes allowed.
+  bool canAggregatePoolStats() const;
+
   // Update stats specific to compact caches
   void updateCompactCacheStats(const std::string& statPrefix,
                                const ICompactCache& c) const;
@@ -342,6 +360,12 @@ class CacheBase {
       poolResizeStrategies_;
   std::shared_ptr<PoolOptimizeStrategy> poolOptimizeStrategy_;
 
+  // Enable aggregating pool stats
+  void enableAggregatePoolStats() { aggregatePoolStats_ = true; }
+
+  // Check if pool stats aggregation is enabled
+  bool isAggregatePoolStatsEnabled() const { return aggregatePoolStats_; }
+
   friend PoolResizer;
   friend PoolRebalancer;
   friend PoolOptimizer;

cachelib/allocator/CacheAllocator.h

@@ -2539,6 +2539,9 @@ CacheAllocator<CacheTrait>::restoreCCacheManager() {
 
 template <typename CacheTrait>
 void CacheAllocator<CacheTrait>::initCommon(bool dramCacheAttached) {
+  // Initialize aggregate pool stats from config
+  aggregatePoolStats_ = config_.isAggregatePoolStatsEnabled();
+
   if (config_.nvmConfig.has_value()) {
     if (config_.nvmCacheAP) {
       nvmAdmissionPolicy_ = config_.nvmCacheAP;

cachelib/allocator/CacheAllocatorConfig.h

@@ -356,6 +356,35 @@ class CacheAllocatorConfig {
     return skipPromoteChildrenWhenParentFailed;
   }
 
+  // Enable aggregating pool stats to a single stat
+  //
+  // When enabled, pool stats from all pools will be aggregated into a single
+  // "aggregated" stat to reduce ODS counter inflation. For example, with two
+  // pools and this option disabled, you will have separate stats like:
+  // -
+  // cachelib.cache_name.pool.cache_name_0.items
+  // -
+  // cachelib.cache_name.pool.cache_name_1.items
+  // With this option enabled, it will be aggregated to:
+  // - cachelib.cache_name.pool.aggregated.items
+  //
+  // LIMITATIONS:
+  // 1. If the cache is using more than 128 distinct allocation sizes across
+  //    all pools, pool stats cannot be aggregated and will fall back to
+  //    separate stat logging.
+  // 2. Some statistics such as evictionAgeSecs (avg and quantiles) may not be
+  //    mathematically precise. These stats are aggregated using weighted
+  //    averages based on the relative number of evictions from each pool.
+  //    While this provides a reasonable approximation, it may not represent
+  //    the exact distribution that would result from treating all pools as
+  //    a single entity.
+  CacheAllocatorConfig& enableAggregatePoolStats();
+
+  // @return whether pool stats aggregation is enabled
+  bool isAggregatePoolStatsEnabled() const noexcept {
+    return aggregatePoolStats;
+  }
+
   // @return whether compact cache is enabled
   bool isCompactCacheEnabled() const noexcept { return enableZeroedSlabAllocs; }
 
@@ -666,6 +695,9 @@ class CacheAllocatorConfig {
 
   size_t numShards{8192};
 
+  // If true, aggregate pool stats into a single stat before exporting
+  bool aggregatePoolStats{false};
+
   friend CacheT;
 
  private:
@@ -1147,6 +1179,12 @@ CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::setNumShards(size_t shards) {
   return *this;
 }
 
+template <typename T>
+CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::enableAggregatePoolStats() {
+  aggregatePoolStats = true;
+  return *this;
+}
+
 template <typename T>
 const CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::validate() const {
   // we can track tail hits only if MMType is MM2Q
@@ -1300,6 +1338,7 @@ std::map<std::string, std::string> CacheAllocatorConfig<T>::serialize() const {
   configMap["delayCacheWorkersStart"] =
       delayCacheWorkersStart ? "true" : "false";
   configMap["numShards"] = std::to_string(numShards);
+  configMap["aggregatePoolStats"] = aggregatePoolStats ? "true" : "false";
   mergeWithPrefix(configMap, throttleConfig.serialize(), "throttleConfig");
   mergeWithPrefix(configMap,
                   chainedItemAccessConfig.serialize(),