Skip to content

[common][dynamicconfig] Move dynamic config to fx Module #6828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/server/cadence/cadence.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

"github.com/uber/cadence/common/client"
"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/dynamicconfig/dynamicconfigfx"
"github.com/uber/cadence/common/log/logfx"
"github.com/uber/cadence/common/service"

Expand Down Expand Up @@ -113,6 +114,7 @@ func BuildCLI(releaseVersion string, gitRevision string) *cli.App {
fxApp := fx.New(
config.Module,
logfx.Module,
dynamicconfigfx.Module,
fx.Provide(func() appContext {
return appContext{
CfgContext: config.Context{
Expand Down
36 changes: 16 additions & 20 deletions cmd/server/cadence/fx.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

"github.com/uber/cadence/common"
"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/dynamicconfig"
"github.com/uber/cadence/common/log"
"github.com/uber/cadence/common/persistence/nosql/nosqlplugin/cassandra/gocql"
"github.com/uber/cadence/tools/cassandra"
Expand All @@ -47,21 +48,21 @@ var Module = fx.Options(
type AppParams struct {
fx.In

RootDir string `name:"root-dir"`
Services []string `name:"services"`
AppContext config.Context
Config config.Config
Logger log.Logger
LifeCycle fx.Lifecycle
Services []string `name:"services"`
AppContext config.Context
Config config.Config
Logger log.Logger
LifeCycle fx.Lifecycle
DynamicConfig dynamicconfig.Client
}

// NewApp created a new Application from pre initalized config and logger.
func NewApp(params AppParams) *App {
app := &App{
cfg: params.Config,
rootDir: params.RootDir,
logger: params.Logger,
services: params.Services,
cfg: params.Config,
logger: params.Logger,
services: params.Services,
dynamicConfig: params.DynamicConfig,
}
params.LifeCycle.Append(fx.Hook{OnStart: app.Start, OnStop: app.Stop})
return app
Expand All @@ -70,21 +71,16 @@ func NewApp(params AppParams) *App {
// App is a fx application that registers itself into fx.Lifecycle and runs.
// It is done implicitly, since it provides methods Start and Stop which are picked up by fx.
type App struct {
cfg config.Config
rootDir string
logger log.Logger
cfg config.Config
rootDir string
logger log.Logger
dynamicConfig dynamicconfig.Client

daemons []common.Daemon
services []string
}

func (a *App) Start(_ context.Context) error {
if a.cfg.DynamicConfig.Client == "" {
a.cfg.DynamicConfigClient.Filepath = constructPathIfNeed(a.rootDir, a.cfg.DynamicConfigClient.Filepath)
} else {
a.cfg.DynamicConfig.FileBased.Filepath = constructPathIfNeed(a.rootDir, a.cfg.DynamicConfig.FileBased.Filepath)
}

if err := a.cfg.ValidateAndFillDefaults(); err != nil {
return fmt.Errorf("config validation failed: %w", err)
}
Expand All @@ -99,7 +95,7 @@ func (a *App) Start(_ context.Context) error {

var daemons []common.Daemon
for _, svc := range a.services {
server := newServer(svc, a.cfg, a.logger)
server := newServer(svc, a.cfg, a.logger, a.dynamicConfig)
daemons = append(daemons, server)
server.Start()
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/server/cadence/fx_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@ import (
"go.uber.org/fx"

"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/dynamicconfig/dynamicconfigfx"
"github.com/uber/cadence/common/log/logfx"
)

func TestFxDependencies(t *testing.T) {
err := fx.ValidateApp(config.Module,
logfx.Module,
dynamicconfigfx.Module,
fx.Provide(func() appContext {
return appContext{
CfgContext: config.Context{
Expand Down
62 changes: 17 additions & 45 deletions cmd/server/cadence/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ import (
"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/constants"
"github.com/uber/cadence/common/dynamicconfig"
"github.com/uber/cadence/common/dynamicconfig/configstore"
"github.com/uber/cadence/common/dynamicconfig/dynamicproperties"
"github.com/uber/cadence/common/elasticsearch"
"github.com/uber/cadence/common/isolationgroup/isolationgroupapi"
Expand All @@ -56,7 +55,6 @@ import (
"github.com/uber/cadence/common/messaging/kafka"
"github.com/uber/cadence/common/metrics"
"github.com/uber/cadence/common/peerprovider/ringpopprovider"
"github.com/uber/cadence/common/persistence"
pnt "github.com/uber/cadence/common/pinot"
"github.com/uber/cadence/common/resource"
"github.com/uber/cadence/common/rpc"
Expand All @@ -75,22 +73,24 @@ import (

type (
server struct {
name string
cfg config.Config
logger log.Logger
doneC chan struct{}
daemon common.Daemon
name string
cfg config.Config
logger log.Logger
doneC chan struct{}
daemon common.Daemon
dynamicCfgClient dynamicconfig.Client
}
)

// newServer returns a new instance of a daemon
// that represents a cadence service
func newServer(service string, cfg config.Config, logger log.Logger) common.Daemon {
func newServer(service string, cfg config.Config, logger log.Logger, dynamicCfgClient dynamicconfig.Client) common.Daemon {
return &server{
cfg: cfg,
name: service,
doneC: make(chan struct{}),
logger: logger.WithTags(tag.Service(service)),
cfg: cfg,
name: service,
doneC: make(chan struct{}),
logger: logger,
dynamicCfgClient: dynamicCfgClient,
}
}

Expand Down Expand Up @@ -125,39 +125,11 @@ func (s *server) startService() common.Daemon {
s.logger.Fatal(err.Error())
}

params := resource.Params{}
params.Name = service.FullName(s.name)

params.Logger = s.logger.WithTags(tag.Service(params.Name))

params.PersistenceConfig = s.cfg.Persistence

err = nil
if s.cfg.DynamicConfig.Client == "" {
params.Logger.Warn("falling back to legacy file based dynamicClientConfig")
params.DynamicConfig, err = dynamicconfig.NewFileBasedClient(&s.cfg.DynamicConfigClient, params.Logger, s.doneC)
} else {
switch s.cfg.DynamicConfig.Client {
case dynamicconfig.ConfigStoreClient:
params.Logger.Info("initialising ConfigStore dynamic config client")
params.DynamicConfig, err = configstore.NewConfigStoreClient(
&s.cfg.DynamicConfig.ConfigStore,
&s.cfg.Persistence,
params.Logger,
persistence.DynamicConfig,
)
case dynamicconfig.FileBasedClient:
params.Logger.Info("initialising File Based dynamic config client")
params.DynamicConfig, err = dynamicconfig.NewFileBasedClient(&s.cfg.DynamicConfig.FileBased, params.Logger, s.doneC)
default:
params.Logger.Info("initialising NOP dynamic config client")
params.DynamicConfig = dynamicconfig.NewNopClient()
}
}

if err != nil {
params.Logger.Error("creating dynamic config client failed, using no-op config client instead", tag.Error(err))
params.DynamicConfig = dynamicconfig.NewNopClient()
params := resource.Params{
Name: service.FullName(s.name),
Logger: s.logger.WithTags(tag.Service(service.FullName(s.name))),
PersistenceConfig: s.cfg.Persistence,
DynamicConfig: s.dynamicCfgClient,
}

clusterGroupMetadata := s.cfg.ClusterGroupMetadata
Expand Down
8 changes: 7 additions & 1 deletion cmd/server/cadence/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,20 @@
package cadence

import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"go.uber.org/fx/fxtest"
"go.uber.org/mock/gomock"

"github.com/uber/cadence/common"
"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/dynamicconfig"
"github.com/uber/cadence/common/dynamicconfig/dynamicconfigfx"
"github.com/uber/cadence/common/dynamicconfig/dynamicproperties"
"github.com/uber/cadence/common/log"
"github.com/uber/cadence/common/log/tag"
Expand Down Expand Up @@ -94,17 +97,20 @@ func (s *ServerSuite) TestServerStartup() {

logger := testlogger.New(s.T())

lifecycle := fxtest.NewLifecycle(s.T())

var daemons []common.Daemon
services := service.ShortNames(service.List)
for _, svc := range services {
server := newServer(svc, cfg, logger)
server := newServer(svc, cfg, logger, dynamicconfigfx.New(dynamicconfigfx.Params{Logger: logger, Cfg: cfg, Lifecycle: lifecycle}))
daemons = append(daemons, server)
server.Start()
}

timer := time.NewTimer(time.Second * 10)

<-timer.C
s.NoError(lifecycle.Stop(context.Background()))
for _, daemon := range daemons {
daemon.Stop()
}
Expand Down
108 changes: 108 additions & 0 deletions common/dynamicconfig/dynamicconfigfx/fx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// The MIT License (MIT)

// Copyright (c) 2017-2020 Uber Technologies Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package dynamicconfigfx

import (
"context"
"path/filepath"

"go.uber.org/fx"

"github.com/uber/cadence/common/config"
"github.com/uber/cadence/common/dynamicconfig"
"github.com/uber/cadence/common/dynamicconfig/configstore"
"github.com/uber/cadence/common/log"
"github.com/uber/cadence/common/log/tag"
"github.com/uber/cadence/common/persistence"
)

// Module provides fx options for dynamic config initialization
var Module = fx.Options(fx.Provide(New))

// Params required to build a new dynamic config.
type Params struct {
fx.In

Cfg config.Config
Logger log.Logger
RootDir string `name:"root-dir"`

Lifecycle fx.Lifecycle
}

// New creates dynamicconfig.Client from the configuration
func New(p Params) dynamicconfig.Client {
stopped := make(chan struct{})

if p.Cfg.DynamicConfig.Client == "" {
p.Cfg.DynamicConfigClient.Filepath = constructPathIfNeed(p.RootDir, p.Cfg.DynamicConfigClient.Filepath)
} else {
p.Cfg.DynamicConfig.FileBased.Filepath = constructPathIfNeed(p.RootDir, p.Cfg.DynamicConfig.FileBased.Filepath)
}

p.Lifecycle.Append(fx.Hook{OnStop: func(_ context.Context) error {
close(stopped)
return nil
}})

var res dynamicconfig.Client

var err error
if p.Cfg.DynamicConfig.Client == "" {
p.Logger.Warn("falling back to legacy file based dynamicClientConfig")
res, err = dynamicconfig.NewFileBasedClient(&p.Cfg.DynamicConfigClient, p.Logger, stopped)
} else {
switch p.Cfg.DynamicConfig.Client {
case dynamicconfig.ConfigStoreClient:
p.Logger.Info("initialising ConfigStore dynamic config client")
res, err = configstore.NewConfigStoreClient(
&p.Cfg.DynamicConfig.ConfigStore,
&p.Cfg.Persistence,
p.Logger,
persistence.DynamicConfig,
)
case dynamicconfig.FileBasedClient:
p.Logger.Info("initialising File Based dynamic config client")
res, err = dynamicconfig.NewFileBasedClient(&p.Cfg.DynamicConfig.FileBased, p.Logger, stopped)
}
}

if res == nil {
p.Logger.Info("initialising NOP dynamic config client")
res = dynamicconfig.NewNopClient()
} else if err != nil {
p.Logger.Error("creating dynamic config client failed, using no-op config client instead", tag.Error(err))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Error cases silently switching to no-op config doesn't sound right. I know that's also the current behavior but we should consider failing startup if there's a misconfiguration for dynamicconfigs.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good point, but I'm unsure how safe it is. It's something to consider for v2: a stricter startup to ensure no missing dependencies.

res = dynamicconfig.NewNopClient()
}

return res
}

// constructPathIfNeed would append the dir as the root dir
// when the file wasn't absolute path.
func constructPathIfNeed(dir string, file string) string {
if !filepath.IsAbs(file) {
return dir + "/" + file
}
return file
}
Loading