Add label size bytes native histogram (#6380)

* Add label size bytes native histogram

Signed-off-by: Anna Tran <[email protected]>

* Update DeletePerUserValidationMetrics with labelSizeBytes

Signed-off-by: Anna Tran <[email protected]>

---------

Signed-off-by: Anna Tran <[email protected]>

Author: anna-tran

CHANGELOG.md

@@ -39,6 +39,7 @@
 * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276
 * [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310
 * [ENHANCEMENT] Distributor: Add new `cortex_distributor_inflight_client_requests` metric to track number of ingester client inflight requests. #6358
+* [ENHANCEMENT] Distributor: Expose `cortex_label_size_bytes` native histogram metric. #6372
 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224
 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326
 * [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled  #6271

pkg/util/validation/validate.go

@@ -78,6 +78,7 @@ type ValidateMetrics struct {
 	DiscardedExemplars                *prometheus.CounterVec
 	DiscardedMetadata                 *prometheus.CounterVec
 	HistogramSamplesReducedResolution *prometheus.CounterVec
+	LabelSizeBytes                    *prometheus.HistogramVec
 }
 
 func registerCollector(r prometheus.Registerer, c prometheus.Collector) {
@@ -120,11 +121,21 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics {
 		[]string{"user"},
 	)
 	registerCollector(r, histogramSamplesReducedResolution)
+	labelSizeBytes := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:                            "cortex_label_size_bytes",
+		Help:                            "The combined size in bytes of all labels and label values for a time series.",
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 1 * time.Hour,
+	}, []string{"user"})
+	registerCollector(r, labelSizeBytes)
+
 	m := &ValidateMetrics{
 		DiscardedSamples:                  discardedSamples,
 		DiscardedExemplars:                discardedExemplars,
 		DiscardedMetadata:                 discardedMetadata,
 		HistogramSamplesReducedResolution: histogramSamplesReducedResolution,
+		LabelSizeBytes:                    labelSizeBytes,
 	}
 
 	return m
@@ -236,6 +247,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str
 		lastLabelName = l.Name
 		labelsSizeBytes += l.Size()
 	}
+	validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes))
 	if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes {
 		validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc()
 		return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes)
@@ -352,4 +364,7 @@ func DeletePerUserValidationMetrics(validateMetrics *ValidateMetrics, userID str
 	if err := util.DeleteMatchingLabels(validateMetrics.HistogramSamplesReducedResolution, filter); err != nil {
 		level.Warn(log).Log("msg", "failed to remove cortex_reduced_resolution_histogram_samples_total metric for user", "user", userID, "err", err)
 	}
+	if err := util.DeleteMatchingLabels(validateMetrics.LabelSizeBytes, filter); err != nil {
+		level.Warn(log).Log("msg", "failed to remove cortex_label_size_bytes metric for user", "user", userID, "err", err)
+	}
 }

pkg/util/validation/validate_test.go

@@ -119,6 +119,14 @@ func TestValidateLabels(t *testing.T) {
 			cortex_discarded_samples_total{reason="random reason",user="different user"} 1
 	`), "cortex_discarded_samples_total"))
 
+	require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
+			# HELP cortex_label_size_bytes The combined size in bytes of all labels and label values for a time series.
+			# TYPE cortex_label_size_bytes histogram
+			cortex_label_size_bytes_bucket{user="testUser",le="+Inf"} 3
+			cortex_label_size_bytes_sum{user="testUser"} 148
+			cortex_label_size_bytes_count{user="testUser"} 3
+	`), "cortex_label_size_bytes"))
+
 	DeletePerUserValidationMetrics(validateMetrics, userID, util_log.Logger)
 
 	require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`