Skip to content

Commit c46aec6

Browse files
alanprotalanprot
authored
Using a single seed array for expanded postings cache on ingesters (#6365)
* Using a single seed array for expanded postings cache on ingesters Signed-off-by: alanprot <[email protected]> * using tenant id to calculate the seeds hash Signed-off-by: alanprot <[email protected]> * Adding cache isolation test Signed-off-by: alanprot <[email protected]> * add test for memHashString Signed-off-by: alanprot <[email protected]> --------- Signed-off-by: alanprot <[email protected]>
1 parent 0dc2e33 commit c46aec6

File tree

4 files changed

+169
-44
lines changed

4 files changed

+169
-44
lines changed

‎pkg/ingester/ingester.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,8 @@ type Ingester struct {
238238

239239
inflightQueryRequests atomic.Int64
240240
maxInflightQueryRequests util_math.MaxTracker
241+
242+
expandedPostingsCacheFactory *cortex_tsdb.ExpandedPostingsCacheFactory
241243
}
242244

243245
// Shipper interface is used to have an easy way to mock it in tests.
@@ -697,12 +699,13 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe
697699
}
698700

699701
i := &Ingester{
700-
cfg: cfg,
701-
limits: limits,
702-
usersMetadata: map[string]*userMetricsMetadata{},
703-
TSDBState: newTSDBState(bucketClient, registerer),
704-
logger: logger,
705-
ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval),
702+
cfg: cfg,
703+
limits: limits,
704+
usersMetadata: map[string]*userMetricsMetadata{},
705+
TSDBState: newTSDBState(bucketClient, registerer),
706+
logger: logger,
707+
ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval),
708+
expandedPostingsCacheFactory: cortex_tsdb.NewExpandedPostingsCacheFactory(cfg.BlocksStorageConfig.TSDB.PostingsCache),
706709
}
707710
i.metrics = newIngesterMetrics(registerer,
708711
false,
@@ -2174,9 +2177,8 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) {
21742177
blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds()
21752178

21762179
var postingCache cortex_tsdb.ExpandedPostingsCache
2177-
if i.cfg.BlocksStorageConfig.TSDB.PostingsCache.Head.Enabled || i.cfg.BlocksStorageConfig.TSDB.PostingsCache.Blocks.Enabled {
2178-
logutil.WarnExperimentalUse("expanded postings cache")
2179-
postingCache = cortex_tsdb.NewBlocksPostingsForMatchersCache(i.cfg.BlocksStorageConfig.TSDB.PostingsCache, i.metrics.expandedPostingsCacheMetrics)
2180+
if i.expandedPostingsCacheFactory != nil {
2181+
postingCache = i.expandedPostingsCacheFactory.NewExpandedPostingsCache(userID, i.metrics.expandedPostingsCacheMetrics)
21802182
}
21812183

21822184
userDB := &userTSDB{

‎pkg/ingester/ingester_test.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5080,6 +5080,65 @@ func TestIngester_instanceLimitsMetrics(t *testing.T) {
50805080
`), "cortex_ingester_instance_limits"))
50815081
}
50825082

5083+
func TestExpendedPostingsCacheIsolation(t *testing.T) {
5084+
cfg := defaultIngesterTestConfig(t)
5085+
cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour}
5086+
cfg.LifecyclerConfig.JoinAfter = 0
5087+
cfg.BlocksStorageConfig.TSDB.PostingsCache = cortex_tsdb.TSDBPostingsCacheConfig{
5088+
SeedSize: 1, // lets make sure all metric names collide
5089+
Head: cortex_tsdb.PostingsCacheConfig{
5090+
Enabled: true,
5091+
},
5092+
Blocks: cortex_tsdb.PostingsCacheConfig{
5093+
Enabled: true,
5094+
},
5095+
}
5096+
5097+
r := prometheus.NewRegistry()
5098+
i, err := prepareIngesterWithBlocksStorage(t, cfg, r)
5099+
require.NoError(t, err)
5100+
require.NoError(t, services.StartAndAwaitRunning(context.Background(), i))
5101+
defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck
5102+
5103+
numberOfTenants := 100
5104+
wg := sync.WaitGroup{}
5105+
wg.Add(numberOfTenants)
5106+
5107+
for j := 0; j < numberOfTenants; j++ {
5108+
go func() {
5109+
defer wg.Done()
5110+
userId := fmt.Sprintf("user%v", j)
5111+
ctx := user.InjectOrgID(context.Background(), userId)
5112+
_, err = i.Push(ctx, cortexpb.ToWriteRequest(
5113+
[]labels.Labels{labels.FromStrings(labels.MetricName, "foo", "userId", userId)}, []cortexpb.Sample{{Value: 2, TimestampMs: 4 * 60 * 60 * 1000}}, nil, nil, cortexpb.API))
5114+
require.NoError(t, err)
5115+
}()
5116+
}
5117+
5118+
wg.Wait()
5119+
5120+
wg.Add(numberOfTenants)
5121+
for j := 0; j < numberOfTenants; j++ {
5122+
go func() {
5123+
defer wg.Done()
5124+
userId := fmt.Sprintf("user%v", j)
5125+
ctx := user.InjectOrgID(context.Background(), userId)
5126+
s := &mockQueryStreamServer{ctx: ctx}
5127+
5128+
err := i.QueryStream(&client.QueryRequest{
5129+
StartTimestampMs: 0,
5130+
EndTimestampMs: math.MaxInt64,
5131+
Matchers: []*client.LabelMatcher{{Type: client.EQUAL, Name: labels.MetricName, Value: "foo"}},
5132+
}, s)
5133+
require.NoError(t, err)
5134+
require.Len(t, s.series, 1)
5135+
require.Len(t, s.series[0].Labels, 2)
5136+
require.Equal(t, userId, cortexpb.FromLabelAdaptersToLabels(s.series[0].Labels).Get("userId"))
5137+
}()
5138+
}
5139+
wg.Wait()
5140+
}
5141+
50835142
func TestExpendedPostingsCache(t *testing.T) {
50845143
cfg := defaultIngesterTestConfig(t)
50855144
cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour}

‎pkg/storage/tsdb/expanded_postings_cache.go

Lines changed: 81 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,17 @@ import (
1010
"sync"
1111
"time"
1212

13-
"github.com/cespare/xxhash/v2"
1413
"github.com/oklog/ulid"
1514
"github.com/prometheus/client_golang/prometheus"
1615
"github.com/prometheus/client_golang/prometheus/promauto"
1716
"github.com/prometheus/prometheus/model/labels"
1817
"github.com/prometheus/prometheus/storage"
1918
"github.com/prometheus/prometheus/tsdb"
2019
"github.com/prometheus/prometheus/tsdb/index"
20+
"github.com/segmentio/fasthash/fnv1a"
2121

2222
"github.com/cortexproject/cortex/pkg/util/extract"
23+
logutil "github.com/cortexproject/cortex/pkg/util/log"
2324
)
2425

2526
var (
@@ -29,8 +30,8 @@ var (
2930

3031
const (
3132
// size of the seed array. Each seed is a 64bits int (8 bytes)
32-
// totaling 8mb
33-
seedArraySize = 1024 * 1024
33+
// totaling 16mb
34+
seedArraySize = 2 * 1024 * 1024
3435

3536
numOfSeedsStripes = 512
3637
)
@@ -67,7 +68,9 @@ type TSDBPostingsCacheConfig struct {
6768
Head PostingsCacheConfig `yaml:"head" doc:"description=If enabled, ingesters will cache expanded postings for the head block. Only queries with with an equal matcher for metric __name__ are cached."`
6869
Blocks PostingsCacheConfig `yaml:"blocks" doc:"description=If enabled, ingesters will cache expanded postings for the compacted blocks. The cache is shared between all blocks."`
6970

71+
// The configurations below are used only for testing purpose
7072
PostingsForMatchers func(ctx context.Context, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) `yaml:"-"`
73+
SeedSize int `yaml:"-"`
7174
timeNow func() time.Time `yaml:"-"`
7275
}
7376

@@ -89,25 +92,48 @@ func (cfg *PostingsCacheConfig) RegisterFlagsWithPrefix(prefix, block string, f
8992
f.BoolVar(&cfg.Enabled, prefix+"expanded_postings_cache."+block+".enabled", false, "Whether the postings cache is enabled or not")
9093
}
9194

95+
type ExpandedPostingsCacheFactory struct {
96+
seedByHash *seedByHash
97+
cfg TSDBPostingsCacheConfig
98+
}
99+
100+
func NewExpandedPostingsCacheFactory(cfg TSDBPostingsCacheConfig) *ExpandedPostingsCacheFactory {
101+
if cfg.Head.Enabled || cfg.Blocks.Enabled {
102+
if cfg.SeedSize == 0 {
103+
cfg.SeedSize = seedArraySize
104+
}
105+
logutil.WarnExperimentalUse("expanded postings cache")
106+
return &ExpandedPostingsCacheFactory{
107+
cfg: cfg,
108+
seedByHash: newSeedByHash(cfg.SeedSize),
109+
}
110+
}
111+
112+
return nil
113+
}
114+
115+
func (f *ExpandedPostingsCacheFactory) NewExpandedPostingsCache(userId string, metrics *ExpandedPostingsCacheMetrics) ExpandedPostingsCache {
116+
return newBlocksPostingsForMatchersCache(userId, f.cfg, metrics, f.seedByHash)
117+
}
118+
92119
type ExpandedPostingsCache interface {
93120
PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error)
94121
ExpireSeries(metric labels.Labels)
95122
}
96123

97-
type BlocksPostingsForMatchersCache struct {
98-
strippedLock []sync.RWMutex
99-
100-
headCache *fifoCache[[]storage.SeriesRef]
101-
blocksCache *fifoCache[[]storage.SeriesRef]
124+
type blocksPostingsForMatchersCache struct {
125+
userId string
102126

103-
headSeedByMetricName []int
127+
headCache *fifoCache[[]storage.SeriesRef]
128+
blocksCache *fifoCache[[]storage.SeriesRef]
104129
postingsForMatchersFunc func(ctx context.Context, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error)
105130
timeNow func() time.Time
106131

107-
metrics *ExpandedPostingsCacheMetrics
132+
metrics *ExpandedPostingsCacheMetrics
133+
seedByHash *seedByHash
108134
}
109135

110-
func NewBlocksPostingsForMatchersCache(cfg TSDBPostingsCacheConfig, metrics *ExpandedPostingsCacheMetrics) ExpandedPostingsCache {
136+
func newBlocksPostingsForMatchersCache(userId string, cfg TSDBPostingsCacheConfig, metrics *ExpandedPostingsCacheMetrics, seedByHash *seedByHash) ExpandedPostingsCache {
111137
if cfg.PostingsForMatchers == nil {
112138
cfg.PostingsForMatchers = tsdb.PostingsForMatchers
113139
}
@@ -116,36 +142,30 @@ func NewBlocksPostingsForMatchersCache(cfg TSDBPostingsCacheConfig, metrics *Exp
116142
cfg.timeNow = time.Now
117143
}
118144

119-
return &BlocksPostingsForMatchersCache{
145+
return &blocksPostingsForMatchersCache{
120146
headCache: newFifoCache[[]storage.SeriesRef](cfg.Head, "head", metrics, cfg.timeNow),
121147
blocksCache: newFifoCache[[]storage.SeriesRef](cfg.Blocks, "block", metrics, cfg.timeNow),
122-
headSeedByMetricName: make([]int, seedArraySize),
123-
strippedLock: make([]sync.RWMutex, numOfSeedsStripes),
124148
postingsForMatchersFunc: cfg.PostingsForMatchers,
125149
timeNow: cfg.timeNow,
126150
metrics: metrics,
151+
seedByHash: seedByHash,
152+
userId: userId,
127153
}
128154
}
129155

130-
func (c *BlocksPostingsForMatchersCache) ExpireSeries(metric labels.Labels) {
156+
func (c *blocksPostingsForMatchersCache) ExpireSeries(metric labels.Labels) {
131157
metricName, err := extract.MetricNameFromLabels(metric)
132158
if err != nil {
133159
return
134160
}
135-
136-
h := MemHashString(metricName)
137-
i := h % uint64(len(c.headSeedByMetricName))
138-
l := h % uint64(len(c.strippedLock))
139-
c.strippedLock[l].Lock()
140-
defer c.strippedLock[l].Unlock()
141-
c.headSeedByMetricName[i]++
161+
c.seedByHash.incrementSeed(c.userId, metricName)
142162
}
143163

144-
func (c *BlocksPostingsForMatchersCache) PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) {
164+
func (c *blocksPostingsForMatchersCache) PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) {
145165
return c.fetchPostings(blockID, ix, ms...)(ctx)
146166
}
147167

148-
func (c *BlocksPostingsForMatchersCache) fetchPostings(blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) func(context.Context) (index.Postings, error) {
168+
func (c *blocksPostingsForMatchersCache) fetchPostings(blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) func(context.Context) (index.Postings, error) {
149169
var seed string
150170
cache := c.blocksCache
151171

@@ -197,7 +217,7 @@ func (c *BlocksPostingsForMatchersCache) fetchPostings(blockID ulid.ULID, ix tsd
197217
return c.result(promise)
198218
}
199219

200-
func (c *BlocksPostingsForMatchersCache) result(ce *cacheEntryPromise[[]storage.SeriesRef]) func(ctx context.Context) (index.Postings, error) {
220+
func (c *blocksPostingsForMatchersCache) result(ce *cacheEntryPromise[[]storage.SeriesRef]) func(ctx context.Context) (index.Postings, error) {
201221
return func(ctx context.Context) (index.Postings, error) {
202222
select {
203223
case <-ctx.Done():
@@ -211,16 +231,11 @@ func (c *BlocksPostingsForMatchersCache) result(ce *cacheEntryPromise[[]storage.
211231
}
212232
}
213233

214-
func (c *BlocksPostingsForMatchersCache) getSeedForMetricName(metricName string) string {
215-
h := MemHashString(metricName)
216-
i := h % uint64(len(c.headSeedByMetricName))
217-
l := h % uint64(len(c.strippedLock))
218-
c.strippedLock[l].RLock()
219-
defer c.strippedLock[l].RUnlock()
220-
return strconv.Itoa(c.headSeedByMetricName[i])
234+
func (c *blocksPostingsForMatchersCache) getSeedForMetricName(metricName string) string {
235+
return c.seedByHash.getSeed(c.userId, metricName)
221236
}
222237

223-
func (c *BlocksPostingsForMatchersCache) cacheKey(seed string, blockID ulid.ULID, ms ...*labels.Matcher) string {
238+
func (c *blocksPostingsForMatchersCache) cacheKey(seed string, blockID ulid.ULID, ms ...*labels.Matcher) string {
224239
slices.SortFunc(ms, func(i, j *labels.Matcher) int {
225240
if i.Type != j.Type {
226241
return int(i.Type - j.Type)
@@ -272,6 +287,36 @@ func metricNameFromMatcher(ms []*labels.Matcher) (string, bool) {
272287
return "", false
273288
}
274289

290+
type seedByHash struct {
291+
strippedLock []sync.RWMutex
292+
seedByHash []int
293+
}
294+
295+
func newSeedByHash(size int) *seedByHash {
296+
return &seedByHash{
297+
seedByHash: make([]int, size),
298+
strippedLock: make([]sync.RWMutex, numOfSeedsStripes),
299+
}
300+
}
301+
302+
func (s *seedByHash) getSeed(userId string, v string) string {
303+
h := memHashString(userId, v)
304+
i := h % uint64(len(s.seedByHash))
305+
l := h % uint64(len(s.strippedLock))
306+
s.strippedLock[l].RLock()
307+
defer s.strippedLock[l].RUnlock()
308+
return strconv.Itoa(s.seedByHash[i])
309+
}
310+
311+
func (s *seedByHash) incrementSeed(userId string, v string) {
312+
h := memHashString(userId, v)
313+
i := h % uint64(len(s.seedByHash))
314+
l := h % uint64(len(s.strippedLock))
315+
s.strippedLock[l].Lock()
316+
defer s.strippedLock[l].Unlock()
317+
s.seedByHash[i]++
318+
}
319+
275320
type fifoCache[V any] struct {
276321
cfg PostingsCacheConfig
277322
cachedValues *sync.Map
@@ -425,6 +470,7 @@ func (ce *cacheEntryPromise[V]) isExpired(ttl time.Duration, now time.Time) bool
425470
return r >= ttl
426471
}
427472

428-
func MemHashString(str string) uint64 {
429-
return xxhash.Sum64(yoloBuf(str))
473+
func memHashString(userId, v string) uint64 {
474+
h := fnv1a.HashString64(userId)
475+
return fnv1a.AddString64(h, v)
430476
}

‎pkg/storage/tsdb/expanded_postings_cache_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,24 @@ func TestFifoCacheExpire(t *testing.T) {
176176
}
177177
}
178178

179+
func Test_memHashString(test *testing.T) {
180+
numberOfTenants := 200
181+
numberOfMetrics := 100
182+
occurrences := map[uint64]int{}
183+
184+
for k := 0; k < 10; k++ {
185+
for j := 0; j < numberOfMetrics; j++ {
186+
metricName := fmt.Sprintf("metricName%v", j)
187+
for i := 0; i < numberOfTenants; i++ {
188+
userId := fmt.Sprintf("user%v", i)
189+
occurrences[memHashString(userId, metricName)]++
190+
}
191+
}
192+
193+
require.Len(test, occurrences, numberOfMetrics*numberOfTenants)
194+
}
195+
}
196+
179197
func RepeatStringIfNeeded(seed string, length int) string {
180198
if len(seed) > length {
181199
return seed

0 commit comments

Comments
 (0)