From b82ce9fa487fa4d82de6879803508dee7c694ba2 Mon Sep 17 00:00:00 2001 From: Wilson La Date: Wed, 22 Apr 2026 11:35:23 -0700 Subject: [PATCH 1/2] Disable bucket index cache if TTL is 0 Signed-off-by: Wilson La --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 2 +- docs/blocks-storage/store-gateway.md | 2 +- docs/configuration/config-file-reference.md | 2 +- pkg/storage/tsdb/caching_bucket.go | 12 +++++++++--- schemas/cortex-config-schema.json | 2 +- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d2be50ef2..837cc7b828a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #TBD * [FEATURE] Distributor: Add experimental `-distributor.enable-start-timestamp` flag for Prometheus Remote Write 2.0. When enabled, `StartTimestamp (ST)` is ingested. #7371 * [FEATURE] Memberlist: Add `-memberlist.cluster-label` and `-memberlist.cluster-label-verification-disabled` to prevent accidental cross-cluster gossip joins and support rolling label rollout. #7385 * [FEATURE] Querier: Add timeout classification to classify query timeouts as 4XX (user error) or 5XX (system error) based on phase timing. When enabled, queries that spend most of their time in PromQL evaluation return `422 Unprocessable Entity` instead of `503 Service Unavailable`. #7374 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 5ceb537848d..57833b875ef 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -1412,7 +1412,7 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: | default = 168h] - # How long to cache content of the bucket index. + # How long to cache content of the bucket index. 0 disables caching # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: | default = 5m] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 22e21cec67f..870773ff3a6 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -1487,7 +1487,7 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: | default = 168h] - # How long to cache content of the bucket index. + # How long to cache content of the bucket index. 0 disables caching # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: | default = 5m] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 2dc7d8bfe12..dc09d0afdff 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2113,7 +2113,7 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: | default = 168h] - # How long to cache content of the bucket index. + # How long to cache content of the bucket index. 0 disables caching # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: | default = 5m] diff --git a/pkg/storage/tsdb/caching_bucket.go b/pkg/storage/tsdb/caching_bucket.go index 2d6b936bed4..d998c6ff53e 100644 --- a/pkg/storage/tsdb/caching_bucket.go +++ b/pkg/storage/tsdb/caching_bucket.go @@ -177,7 +177,7 @@ func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix f.IntVar(&cfg.MetafileMaxSize, prefix+"metafile-max-size-bytes", 1*1024*1024, "Maximum size of metafile content to cache in bytes. Caching will be skipped if the content exceeds this size. This is useful to avoid network round trip for large content if the configured caching backend has an hard limit on cached items size (in this case, you should set this limit to the same limit in the caching backend).") f.DurationVar(&cfg.MetafileAttributesTTL, prefix+"metafile-attributes-ttl", 168*time.Hour, "How long to cache attributes of the block metafile.") f.DurationVar(&cfg.BlockIndexAttributesTTL, prefix+"block-index-attributes-ttl", 168*time.Hour, "How long to cache attributes of the block index.") - f.DurationVar(&cfg.BucketIndexContentTTL, prefix+"bucket-index-content-ttl", 5*time.Minute, "How long to cache content of the bucket index.") + f.DurationVar(&cfg.BucketIndexContentTTL, prefix+"bucket-index-content-ttl", 5*time.Minute, "How long to cache content of the bucket index. 0 disables caching") f.IntVar(&cfg.BucketIndexMaxSize, prefix+"bucket-index-max-size-bytes", 1*1024*1024, "Maximum size of bucket index content to cache in bytes. Caching will be skipped if the content exceeds this size. This is useful to avoid network round trip for large content if the configured caching backend has an hard limit on cached items size (in this case, you should set this limit to the same limit in the caching backend).") f.DurationVar(&cfg.PartitionedGroupsListTTL, prefix+"partitioned-groups-list-ttl", 0, "How long to cache list of partitioned groups for an user. 0 disables caching") } @@ -245,7 +245,10 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata cfg.CacheGet("metafile", metadataCache, matchers.GetMetafileMatcher(), metadataConfig.MetafileMaxSize, metadataConfig.MetafileContentTTL, metadataConfig.MetafileExistsTTL, metadataConfig.MetafileDoesntExistTTL) cfg.CacheAttributes("metafile", metadataCache, matchers.GetMetafileMatcher(), metadataConfig.MetafileAttributesTTL) cfg.CacheAttributes("block-index", metadataCache, matchers.GetBlockIndexMatcher(), metadataConfig.BlockIndexAttributesTTL) - cfg.CacheGet("bucket-index", metadataCache, matchers.GetBucketIndexMatcher(), metadataConfig.BucketIndexMaxSize, metadataConfig.BucketIndexContentTTL /* do not cache exist / not exist: */, 0, 0) + + if metadataConfig.BucketIndexContentTTL > 0 { + cfg.CacheGet("bucket-index", metadataCache, matchers.GetBucketIndexMatcher(), metadataConfig.BucketIndexMaxSize, metadataConfig.BucketIndexContentTTL /* do not cache exist / not exist: */, 0, 0) + } codec := snappyIterCodec{storecache.JSONIterCodec{}} cfg.CacheIter("tenants-iter", metadataCache, matchers.GetTenantsIterMatcher(), metadataConfig.TenantsListTTL, codec, "") @@ -296,8 +299,11 @@ func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner if !cleaner { cfg.CacheExists("metafile", metadataCache, matchers.GetMetafileMatcher(), metadataConfig.MetafileExistsTTL, metadataConfig.MetafileDoesntExistTTL) cfg.CacheGet("metafile", metadataCache, matchers.GetMetafileMatcher(), metadataConfig.MetafileMaxSize, metadataConfig.MetafileContentTTL, metadataConfig.MetafileExistsTTL, metadataConfig.MetafileDoesntExistTTL) - cfg.CacheGet("bucket-index", metadataCache, matchers.GetBucketIndexMatcher(), metadataConfig.BucketIndexMaxSize, metadataConfig.BucketIndexContentTTL /* do not cache exist / not exist: */, 0, 0) cfg.CacheIter("tenant-blocks-iter", metadataCache, matchers.GetTenantBlocksIterMatcher(), metadataConfig.TenantBlocksListTTL, codec, "") + + if metadataConfig.BucketIndexContentTTL > 0 { + cfg.CacheGet("bucket-index", metadataCache, matchers.GetBucketIndexMatcher(), metadataConfig.BucketIndexMaxSize, metadataConfig.BucketIndexContentTTL /* do not cache exist / not exist: */, 0, 0) + } } else { // Cache only GET for metadata and don't cache exists and not exists. cfg.CacheGet("metafile", metadataCache, matchers.GetMetafileMatcher(), metadataConfig.MetafileMaxSize, metadataConfig.MetafileContentTTL, 0, 0) diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index feb746bdb22..16ee201dc62 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -1888,7 +1888,7 @@ }, "bucket_index_content_ttl": { "default": "5m0s", - "description": "How long to cache content of the bucket index.", + "description": "How long to cache content of the bucket index. 0 disables caching", "type": "string", "x-cli-flag": "blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl", "x-format": "duration" From 22c26e5e94b6384097c9d854b4e59ee5dfacca4e Mon Sep 17 00:00:00 2001 From: Wilson La Date: Wed, 22 Apr 2026 13:52:51 -0700 Subject: [PATCH 2/2] Update changelog PR number Signed-off-by: Wilson La --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837cc7b828a..65f3aef0b18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## master / unreleased -* [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #TBD +* [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 * [FEATURE] Distributor: Add experimental `-distributor.enable-start-timestamp` flag for Prometheus Remote Write 2.0. When enabled, `StartTimestamp (ST)` is ingested. #7371 * [FEATURE] Memberlist: Add `-memberlist.cluster-label` and `-memberlist.cluster-label-verification-disabled` to prevent accidental cross-cluster gossip joins and support rolling label rollout. #7385 * [FEATURE] Querier: Add timeout classification to classify query timeouts as 4XX (user error) or 5XX (system error) based on phase timing. When enabled, queries that spend most of their time in PromQL evaluation return `422 Unprocessable Entity` instead of `503 Service Unavailable`. #7374