Add badger index cache

.gitignore

@@ -13,6 +13,8 @@ website/content/en/docs
 e2e_integration_test*
 active-query-tracker
 dist/
+/pkg/storage/tsdb/index-cache/
+
 
 # Binaries built from ./cmd
 /blocksconvert

CHANGELOG.md

@@ -16,6 +16,7 @@
 * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245
 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249
 * [FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. #6256
+* [FEATURE] Index Cache: Experimental: Add a badgerDB for disk index cache. #6258
 * [ENHANCEMENT] Query Frontend/Querier: Add an experimental flag `-querier.enable-promql-experimental-functions` to enable experimental promQL functions. #6355
 * [ENHANCEMENT] OTLP: Add `-distributor.otlp-max-recv-msg-size` flag to limit OTLP request size in bytes. #6333
 * [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280

docs/blocks-storage/querier.md

@@ -590,7 +590,7 @@ blocks_storage:
     index_cache:
       # The index cache backend type. Multiple cache backend can be provided as
       # a comma-separated ordered list to enable the implementation of a cache
-      # hierarchy. Supported values: inmemory, memcached, redis.
+      # hierarchy. Supported values: inmemory, badger, memcached, redis.
       # CLI flag: -blocks-storage.bucket-store.index-cache.backend
       [backend: <string> | default = "inmemory"]
 
@@ -605,6 +605,24 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
         [enabled_items: <list of string> | default = []]
 
+      badger:
+        # [Experimental] Data directory in which to cache index data.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+        [data_dir: <string> | default = "./badger-index-cache"]
+
+        # [Experimental] Selectively cache index item types. Supported values
+        # are Postings, ExpandedPostings and Series.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+        [enabled_items: <list of string> | default = []]
+
+        # [Experimental] Threshold to trigger value log GC of the Badger DB
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+        [gc_threshold: <int> | default = 134217728]
+
+        # [Experimental] Badger DB garbage collection interval.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+        [gc_interval: <duration> | default = 5m]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV

docs/blocks-storage/store-gateway.md

@@ -677,7 +677,7 @@ blocks_storage:
     index_cache:
       # The index cache backend type. Multiple cache backend can be provided as
       # a comma-separated ordered list to enable the implementation of a cache
-      # hierarchy. Supported values: inmemory, memcached, redis.
+      # hierarchy. Supported values: inmemory, badger, memcached, redis.
       # CLI flag: -blocks-storage.bucket-store.index-cache.backend
       [backend: <string> | default = "inmemory"]
 
@@ -692,6 +692,24 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
         [enabled_items: <list of string> | default = []]
 
+      badger:
+        # [Experimental] Data directory in which to cache index data.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+        [data_dir: <string> | default = "./badger-index-cache"]
+
+        # [Experimental] Selectively cache index item types. Supported values
+        # are Postings, ExpandedPostings and Series.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+        [enabled_items: <list of string> | default = []]
+
+        # [Experimental] Threshold to trigger value log GC of the Badger DB
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+        [gc_threshold: <int> | default = 134217728]
+
+        # [Experimental] Badger DB garbage collection interval.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+        [gc_interval: <duration> | default = 5m]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV

docs/configuration/config-file-reference.md

@@ -1131,7 +1131,7 @@ bucket_store:
   index_cache:
     # The index cache backend type. Multiple cache backend can be provided as a
     # comma-separated ordered list to enable the implementation of a cache
-    # hierarchy. Supported values: inmemory, memcached, redis.
+    # hierarchy. Supported values: inmemory, badger, memcached, redis.
     # CLI flag: -blocks-storage.bucket-store.index-cache.backend
     [backend: <string> | default = "inmemory"]
 
@@ -1146,6 +1146,24 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
       [enabled_items: <list of string> | default = []]
 
+    badger:
+      # [Experimental] Data directory in which to cache index data.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+      [data_dir: <string> | default = "./badger-index-cache"]
+
+      # [Experimental] Selectively cache index item types. Supported values are
+      # Postings, ExpandedPostings and Series.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+      [enabled_items: <list of string> | default = []]
+
+      # [Experimental] Threshold to trigger value log GC of the Badger DB
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+      [gc_threshold: <int> | default = 134217728]
+
+      # [Experimental] Badger DB garbage collection interval.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+      [gc_interval: <duration> | default = 5m]
+
     memcached:
       # Comma separated list of memcached addresses. Supported prefixes are:
       # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,

go.mod

@@ -79,10 +79,12 @@ require (
 	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
 	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/dgraph-io/badger/v4 v4.3.1
 	github.com/google/go-cmp v0.6.0
 	github.com/sercand/kuberesolver/v4 v4.0.0
 	go.opentelemetry.io/collector/pdata v1.20.0
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
+	golang.org/x/sys v0.27.0
 	google.golang.org/protobuf v1.35.2
 )
 
@@ -121,6 +123,7 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
+	github.com/dgraph-io/ristretto v1.0.0 // indirect
 	github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -147,6 +150,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -230,7 +234,6 @@ require (
 	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sys v0.27.0 // indirect
 	golang.org/x/text v0.20.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect

go.sum

@@ -953,6 +953,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
+github.com/dgraph-io/badger/v4 v4.3.1 h1:7r5wKqmoRpGgSxqa0S/nGdpOpvvzuREGPLSua73C8tw=
+github.com/dgraph-io/badger/v4 v4.3.1/go.mod h1:oObz97DImXpd6O/Dt8BqdKLLTDmEmarAimo72VV5whQ=
+github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84=
+github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E=
 github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@@ -1163,6 +1169,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
+github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

integration/querier_test.go

@@ -58,6 +58,28 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendRedis),
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
 		},
+		"blocks default sharding, badger index cache": {
+			blocksShardingStrategy: "default",
+			indexCacheBackend:      tsdb.IndexCacheBackendBadger,
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks default sharding, multilevel index cache (inmemory, badger)": {
+			blocksShardingStrategy: "default",
+			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendBadger),
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks shuffle sharding, badger index cache": {
+			blocksShardingStrategy: "default",
+			tenantShardSize:        1,
+			indexCacheBackend:      tsdb.IndexCacheBackendBadger,
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks shuffle sharding, multilevel index cache (inmemory, badger)": {
+			blocksShardingStrategy: "default",
+			tenantShardSize:        1,
+			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendBadger),
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
 		"blocks default sharding, inmemory index cache": {
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendInMemory,