Add badger index cache

Signed-off-by: SungJin1212 <[email protected]>

Author: SungJin1212

.gitignore

@@ -13,6 +13,8 @@ website/content/en/docs
 e2e_integration_test*
 active-query-tracker
 dist/
+/pkg/storage/tsdb/index-cache/
+
 
 # Binaries built from ./cmd
 /blocksconvert

CHANGELOG.md

@@ -9,6 +9,7 @@
 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
 * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245
 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249
+* [FEATURE] Index Cache: Experimental: Add a badgerDB for disk index cache. #6258
 * [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280
 * [ENHANCEMENT] Query Frontend: Add new query stats metrics `cortex_query_samples_scanned_total` and `cortex_query_peak_samples` to track scannedSamples and peakSample per user. #6228
 * [ENHANCEMENT] Ingester: Add `blocks-storage.tsdb.wal-compression-type` to support zstd wal compression type. #6232

docs/blocks-storage/querier.md

@@ -574,7 +574,7 @@ blocks_storage:
     index_cache:
       # The index cache backend type. Multiple cache backend can be provided as
       # a comma-separated ordered list to enable the implementation of a cache
-      # hierarchy. Supported values: inmemory, memcached, redis.
+      # hierarchy. Supported values: inmemory, badger, memcached, redis.
       # CLI flag: -blocks-storage.bucket-store.index-cache.backend
       [backend: <string> | default = "inmemory"]
 
@@ -589,6 +589,24 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
         [enabled_items: <list of string> | default = []]
 
+      badger:
+        # [Experimental] Data directory in which to cache index data.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+        [data_dir: <string> | default = "./badger-index-cache"]
+
+        # [Experimental] Selectively cache index item types. Supported values
+        # are Postings, ExpandedPostings and Series.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+        [enabled_items: <list of string> | default = []]
+
+        # [Experimental] Threshold to trigger value log GC of the Badger DB
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+        [gc_threshold: <int> | default = 134217728]
+
+        # [Experimental] Badger DB garbage collection interval.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+        [gc_interval: <duration> | default = 5m]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV

docs/blocks-storage/store-gateway.md

@@ -665,7 +665,7 @@ blocks_storage:
     index_cache:
       # The index cache backend type. Multiple cache backend can be provided as
       # a comma-separated ordered list to enable the implementation of a cache
-      # hierarchy. Supported values: inmemory, memcached, redis.
+      # hierarchy. Supported values: inmemory, badger, memcached, redis.
       # CLI flag: -blocks-storage.bucket-store.index-cache.backend
       [backend: <string> | default = "inmemory"]
 
@@ -680,6 +680,24 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
         [enabled_items: <list of string> | default = []]
 
+      badger:
+        # [Experimental] Data directory in which to cache index data.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+        [data_dir: <string> | default = "./badger-index-cache"]
+
+        # [Experimental] Selectively cache index item types. Supported values
+        # are Postings, ExpandedPostings and Series.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+        [enabled_items: <list of string> | default = []]
+
+        # [Experimental] Threshold to trigger value log GC of the Badger DB
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+        [gc_threshold: <int> | default = 134217728]
+
+        # [Experimental] Badger DB garbage collection interval.
+        # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+        [gc_interval: <duration> | default = 5m]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV

docs/configuration/config-file-reference.md

@@ -1107,7 +1107,7 @@ bucket_store:
   index_cache:
     # The index cache backend type. Multiple cache backend can be provided as a
     # comma-separated ordered list to enable the implementation of a cache
-    # hierarchy. Supported values: inmemory, memcached, redis.
+    # hierarchy. Supported values: inmemory, badger, memcached, redis.
     # CLI flag: -blocks-storage.bucket-store.index-cache.backend
     [backend: <string> | default = "inmemory"]
 
@@ -1122,6 +1122,24 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
       [enabled_items: <list of string> | default = []]
 
+    badger:
+      # [Experimental] Data directory in which to cache index data.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.data-dir
+      [data_dir: <string> | default = "./badger-index-cache"]
+
+      # [Experimental] Selectively cache index item types. Supported values are
+      # Postings, ExpandedPostings and Series.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.enabled-items
+      [enabled_items: <list of string> | default = []]
+
+      # [Experimental] Threshold to trigger value log GC of the Badger DB
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-threshold
+      [gc_threshold: <int> | default = 134217728]
+
+      # [Experimental] Badger DB garbage collection interval.
+      # CLI flag: -blocks-storage.bucket-store.index-cache.badger.gc-interval
+      [gc_interval: <duration> | default = 5m]
+
     memcached:
       # Comma separated list of memcached addresses. Supported prefixes are:
       # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/hashicorp/go-sockaddr v1.0.6
 	github.com/hashicorp/memberlist v0.5.1
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.17.9
+	github.com/klauspost/compress v1.17.10
 	github.com/lib/pq v1.10.9
 	github.com/minio/minio-go/v7 v7.0.75
 	github.com/mitchellh/go-wordwrap v1.0.1
@@ -79,10 +79,12 @@ require (
 	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3
 	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/dgraph-io/badger/v4 v4.3.1
 	github.com/google/go-cmp v0.6.0
 	github.com/sercand/kuberesolver/v4 v4.0.0
 	go.opentelemetry.io/collector/pdata v1.14.1
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
+	golang.org/x/sys v0.26.0
 	google.golang.org/protobuf v1.34.2
 )
 
@@ -121,6 +123,7 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
+	github.com/dgraph-io/ristretto v1.0.0 // indirect
 	github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -147,6 +150,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -230,7 +234,6 @@ require (
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect

go.sum

@@ -953,6 +953,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
+github.com/dgraph-io/badger/v4 v4.3.1 h1:7r5wKqmoRpGgSxqa0S/nGdpOpvvzuREGPLSua73C8tw=
+github.com/dgraph-io/badger/v4 v4.3.1/go.mod h1:oObz97DImXpd6O/Dt8BqdKLLTDmEmarAimo72VV5whQ=
+github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84=
+github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E=
 github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@@ -1163,6 +1169,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
+github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -1376,8 +1384,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
+github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=

integration/querier_test.go

@@ -58,6 +58,28 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendRedis),
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
 		},
+		"blocks default sharding, badger index cache": {
+			blocksShardingStrategy: "default",
+			indexCacheBackend:      tsdb.IndexCacheBackendBadger,
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks default sharding, multilevel index cache (inmemory, badger)": {
+			blocksShardingStrategy: "default",
+			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendBadger),
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks shuffle sharding, badger index cache": {
+			blocksShardingStrategy: "default",
+			tenantShardSize:        1,
+			indexCacheBackend:      tsdb.IndexCacheBackendBadger,
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
+		"blocks shuffle sharding, multilevel index cache (inmemory, badger)": {
+			blocksShardingStrategy: "default",
+			tenantShardSize:        1,
+			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendBadger),
+			chunkCacheBackend:      tsdb.CacheBackendMemcached,
+		},
 		"blocks default sharding, inmemory index cache": {
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendInMemory,