refactor: refactor metronome metrics with consistent tags

server/config/options.go

@@ -162,6 +162,7 @@ type MetricsConfig struct {
 	Auth              AuthMetricsConfig           `mapstructure:"auth" yaml:"auth" json:"auth"`
 	SecondaryIndex    SecondaryIndexMetricsConfig `mapstructure:"secondary_index" yaml:"secondary_index" json:"secondary_index"`
 	Queue             QueueMetricsConfig          `mapstructure:"queue" yaml:"queue" json:"queue"`
+	Metronome         MetronomeMetricsConfig      `mapstructure:"metronome" yaml:"metronome" json:"metronome"`
 }
 
 type LongRequestConfig struct {
@@ -231,6 +232,13 @@ type SecondaryIndexMetricsConfig struct {
 	FilteredTags []string      `mapstructure:"filtered_tags" yaml:"filtered_tags" json:"filtered_tags"`
 }
 
+type MetronomeMetricsConfig struct {
+	Enabled      bool          `mapstructure:"enabled" yaml:"enabled" json:"enabled"`
+	Counter      CounterConfig `mapstructure:"counter" yaml:"counter" json:"counter"`
+	Timer        TimerConfig   `mapstructure:"timer" yaml:"timer" json:"timer"`
+	FilteredTags []string      `mapstructure:"filtered_tags" yaml:"filtered_tags" json:"filtered_tags"`
+}
+
 type QueueMetricsConfig struct {
 	Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"`
 }
@@ -449,6 +457,17 @@ var DefaultConfig = Config{
 			},
 			FilteredTags: nil,
 		},
+		Metronome: MetronomeMetricsConfig{
+			Enabled: true,
+			Counter: CounterConfig{
+				OkEnabled:    true,
+				ErrorEnabled: true,
+			},
+			Timer: TimerConfig{
+				TimerEnabled:     true,
+				HistogramEnabled: false,
+			},
+		},
 		Session: SessionMetricGroupConfig{
 			Enabled: true,
 			Counter: CounterConfig{

server/metrics/measurement.go

@@ -220,6 +220,10 @@ func (m *Measurement) GetFdbErrorTags(err error) map[string]string {
 	return filterTags(standardizeTags(mergeTags(m.tags, getTagsForError(err)), getFdbErrorTagKeys()), config.DefaultConfig.Metrics.Fdb.FilteredTags)
 }
 
+func (m *Measurement) GetMetronomeTags() map[string]string {
+	return filterTags(standardizeTags(m.tags, getMetronomeTagKeys()), config.DefaultConfig.Metrics.Metronome.FilteredTags)
+}
+
 func (m *Measurement) GetSearchOkTags() map[string]string {
 	return filterTags(standardizeTags(m.tags, getSearchOkTagKeys()), config.DefaultConfig.Metrics.Search.FilteredTags)
 }
@@ -420,10 +424,9 @@ func (m *Measurement) RecordDuration(scope tally.Scope, tags map[string]string)
 	case SecondaryIndexRespTime, SecondaryIndexErrorRespTime:
 		timerEnabled = cfg.SecondaryIndex.Timer.TimerEnabled
 		histogramEnabled = cfg.SecondaryIndex.Timer.HistogramEnabled
-	case MetronomeCreateAccount, MetronomeAddPlan, MetronomeIngest, MetronomeGetInvoice, MetronomeListInvoices,
-		MetronomeGetUsage, MetronomeGetCustomer:
-		timerEnabled = true
-		histogramEnabled = true
+	case MetronomeResponseTime, MetronomeErrorResponseTime:
+		timerEnabled = cfg.Metronome.Timer.TimerEnabled
+		histogramEnabled = cfg.Metronome.Timer.HistogramEnabled
 	case RequestsRespTimeToFirstDoc:
 		// Response time to first document
 		m.RecordFirstDocumentDuration(scope, tags)

server/metrics/metrics.go

@@ -16,6 +16,7 @@ package metrics
 
 import (
 	"io"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -41,6 +42,7 @@ var (
 	SchemaMetrics         tally.Scope
 	MetronomeMetrics      tally.Scope
 	GlobalSt              *GlobalStatus
+	once                  sync.Once
 )
 
 func getVersion() string {
@@ -94,74 +96,76 @@ func SchemaUpdateRepaired(project string, branch string, collection string) {
 
 func InitializeMetrics() func() {
 	var closer io.Closer
-	if cfg := config.DefaultConfig.Metrics; cfg.Enabled {
-		log.Debug().Msg("Initializing metrics")
-		Reporter = promreporter.NewReporter(promreporter.Options{
-			DefaultSummaryObjectives: getTimerSummaryObjectives(),
-		})
-		root, closer = tally.NewRootScope(tally.ScopeOptions{
-			Tags:           GetGlobalTags(),
-			CachedReporter: Reporter,
-			// Panics with .
-			Separator: promreporter.DefaultSeparator,
-		}, 1*time.Second)
-
-		if cfg.Requests.Enabled {
-			// Request level metrics (HTTP and GRPC)
-			Requests = root.SubScope("requests")
-			initializeRequestScopes()
-		}
-		if cfg.Fdb.Enabled {
-			// FDB level metrics
-			FdbMetrics = root.SubScope("fdb")
-			initializeFdbScopes()
-		}
-		if cfg.Search.Enabled {
-			// Search level metrics
-			SearchMetrics = root.SubScope("search")
-			initializeSearchScopes()
-		}
-		if cfg.Session.Enabled {
-			// Session level metrics
-			SessionMetrics = root.SubScope("session")
-			initializeSessionScopes()
-		}
-		if cfg.Size.Enabled {
-			// Size metrics
-			SizeMetrics = root.SubScope("size")
-			initializeSizeScopes()
-		}
-		if cfg.Network.Enabled {
-			// Network metrics
-			NetworkMetrics = root.SubScope("net")
-			initializeNetworkScopes()
-		}
-		if cfg.Auth.Enabled {
-			// Auth metrics
-			AuthMetrics = root.SubScope("auth")
-			initializeAuthScopes()
-		}
+	once.Do(func() {
+		if cfg := config.DefaultConfig.Metrics; cfg.Enabled {
+			log.Debug().Msg("Initializing metrics")
+			Reporter = promreporter.NewReporter(promreporter.Options{
+				DefaultSummaryObjectives: getTimerSummaryObjectives(),
+			})
+			root, closer = tally.NewRootScope(tally.ScopeOptions{
+				Tags:           GetGlobalTags(),
+				CachedReporter: Reporter,
+				// Panics with .
+				Separator: promreporter.DefaultSeparator,
+			}, 1*time.Second)
+
+			if cfg.Requests.Enabled {
+				// Request level metrics (HTTP and GRPC)
+				Requests = root.SubScope("requests")
+				initializeRequestScopes()
+			}
+			if cfg.Fdb.Enabled {
+				// FDB level metrics
+				FdbMetrics = root.SubScope("fdb")
+				initializeFdbScopes()
+			}
+			if cfg.Search.Enabled {
+				// Search level metrics
+				SearchMetrics = root.SubScope("search")
+				initializeSearchScopes()
+			}
+			if cfg.Session.Enabled {
+				// Session level metrics
+				SessionMetrics = root.SubScope("session")
+				initializeSessionScopes()
+			}
+			if cfg.Size.Enabled {
+				// Size metrics
+				SizeMetrics = root.SubScope("size")
+				initializeSizeScopes()
+			}
+			if cfg.Network.Enabled {
+				// Network metrics
+				NetworkMetrics = root.SubScope("net")
+				initializeNetworkScopes()
+			}
+			if cfg.Auth.Enabled {
+				// Auth metrics
+				AuthMetrics = root.SubScope("auth")
+				initializeAuthScopes()
+			}
 
-		if cfg.SecondaryIndex.Enabled {
-			// Secondary Index metrics
-			SecondaryIndexMetrics = root.SubScope("secondary_index")
-			initializeSecondaryIndexScopes()
-		}
+			if cfg.SecondaryIndex.Enabled {
+				// Secondary Index metrics
+				SecondaryIndexMetrics = root.SubScope("secondary_index")
+				initializeSecondaryIndexScopes()
+			}
 
-		if cfg.Queue.Enabled {
-			QueueMetrics = root.SubScope("queue")
-			initializeQueueScopes()
-		}
+			if cfg.Queue.Enabled {
+				QueueMetrics = root.SubScope("queue")
+				initializeQueueScopes()
+			}
 
-		// Metrics for Metronome - external billing service
-		MetronomeMetrics = root.SubScope("metronome")
-		initializeMetronomeScopes()
+			// Metrics for Metronome - external billing service
+			MetronomeMetrics = root.SubScope("metronome")
+			initializeMetronomeScopes()
 
-		initializeQuotaScopes()
+			initializeQuotaScopes()
 
-		SchemaMetrics = root.SubScope("schema")
-		GlobalSt = NewGlobalStatus()
-	}
+			SchemaMetrics = root.SubScope("schema")
+			GlobalSt = NewGlobalStatus()
+		}
+	})
 
 	return func() {
 		if closer != nil {

server/metrics/metronome.go

@@ -17,42 +17,48 @@ package metrics
 import (
 	"strconv"
 
+	"github.com/tigrisdata/tigris/server/config"
 	"github.com/uber-go/tally"
 )
 
 var (
-	MetronomeCreateAccount tally.Scope
-	MetronomeAddPlan       tally.Scope
-	MetronomeIngest        tally.Scope
-	MetronomeListInvoices  tally.Scope
-	MetronomeGetInvoice    tally.Scope
-	MetronomeGetUsage      tally.Scope
-	MetronomeGetCustomer   tally.Scope
+	MetronomeRequestOk         tally.Scope
+	MetronomeRequestError      tally.Scope
+	MetronomeResponseTime      tally.Scope
+	MetronomeErrorResponseTime tally.Scope
+	MetronomeIngest            tally.Scope
 )
 
 func initializeMetronomeScopes() {
-	MetronomeCreateAccount = MetronomeMetrics.SubScope("create_account")
-	MetronomeAddPlan = MetronomeMetrics.SubScope("add_plan")
+	MetronomeRequestOk = MetronomeMetrics.SubScope("count")
+	MetronomeRequestError = MetronomeMetrics.SubScope("count")
+	MetronomeResponseTime = MetronomeMetrics.SubScope("response")
+	MetronomeErrorResponseTime = MetronomeMetrics.SubScope("error_response")
 	MetronomeIngest = MetronomeMetrics.SubScope("ingest")
-	MetronomeListInvoices = MetronomeMetrics.SubScope("list_invoices")
-	MetronomeGetInvoice = MetronomeMetrics.SubScope("get_invoice")
-	MetronomeGetUsage = MetronomeMetrics.SubScope("get_usage")
-	MetronomeGetCustomer = MetronomeMetrics.SubScope("get_customer")
 }
 
-func GetResponseCodeTags(code int) map[string]string {
+func getMetronomeTagKeys() []string {
+	return []string{
+		"env",
+		"operation",
+		"response_code",
+	}
+}
+
+func GetMetronomeBaseTags(operation string) map[string]string {
 	return map[string]string{
-		"response_code": strconv.Itoa(code),
+		"env":       config.GetEnvironment(),
+		"operation": operation,
 	}
 }
 
-func GetErrorCodeTags(err error) map[string]string {
+func GetMetronomeResponseCodeTags(code int) map[string]string {
 	return map[string]string{
-		"error_value": err.Error(),
+		"response_code": strconv.Itoa(code),
 	}
 }
 
-func GetIngestEventTags(eventType string) map[string]string {
+func GetMetronomeIngestEventTags(eventType string) map[string]string {
 	return map[string]string{
 		"event_type": eventType,
 	}

server/metrics/metronome_test.go

@@ -15,14 +15,33 @@
 package metrics
 
 import (
+	"context"
 	"testing"
 )
 
 func TestMetronomeMetrics(t *testing.T) {
 	InitializeMetrics()
-	t.Run("Increment metronome counter", func(t *testing.T) {
-		tags := GetResponseCodeTags(500)
-		MetronomeListInvoices.Tagged(tags).Counter("request").Inc(1)
-		MetronomeGetInvoice.Tagged(tags).Counter("error").Inc(1)
+	t.Run("Test measuring metronome ok requests", func(t *testing.T) {
+		InitializeMetrics()
+		ctx := context.TODO()
+		op := "test_operation"
+		me := NewMeasurement(MetronomeServiceName, op, MetronomeSpanType, GetMetronomeBaseTags(op))
+		me.StartTracing(ctx, false)
+		me.AddTags(GetMetronomeResponseCodeTags(200))
+		me.FinishTracing(ctx)
+		me.CountOkForScope(MetronomeRequestOk, me.GetMetronomeTags())
+		me.RecordDuration(MetronomeResponseTime, me.GetMetronomeTags())
+	})
+
+	t.Run("Test measuring metronome error requests", func(t *testing.T) {
+		InitializeMetrics()
+		ctx := context.TODO()
+		op := "test_operation"
+		me := NewMeasurement(MetronomeServiceName, op, MetronomeSpanType, GetMetronomeBaseTags(op))
+		me.StartTracing(ctx, false)
+		me.AddTags(GetMetronomeResponseCodeTags(500))
+		me.FinishTracing(ctx)
+		me.CountErrorForScope(MetronomeRequestError, me.GetMetronomeTags())
+		me.RecordDuration(MetronomeErrorResponseTime, me.GetMetronomeTags())
 	})
 }