Skip to content

✨ feat: add support for custom labels on leader election leases ✨ #3237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions pkg/leaderelection/leader_election.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,17 @@ type Options struct {
// Without that, a single slow response from the API server can result
// in losing leadership.
RenewDeadline time.Duration

// LeaderLabels are an optional set of labels that will be set on the lease object
// when this replica becomes leader
LeaderLabels map[string]string
}

// NewResourceLock creates a new resource lock for use in a leader election loop.
func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, options Options) (resourcelock.Interface, error) {
if !options.LeaderElection {
return nil, nil
}

// Default resource lock to "leases". The previous default (from v0.7.0 to v0.11.x) was configmapsleases, which was
// used to migrate from configmaps to leases. Since the default was "configmapsleases" for over a year, spanning
// five minor releases, any actively maintained operators are very likely to have a released version that uses
Expand Down Expand Up @@ -93,22 +96,21 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
}
id = id + "_" + string(uuid.NewUUID())

// Construct clients for leader election
rest.AddUserAgent(config, "leader-election")
// Construct config for leader election
config = rest.AddUserAgent(config, "leader-election")

// Timeout set for a client used to contact to Kubernetes should be lower than
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this have to do with the rest of the PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This lets controllers that have RenewDeadline set also use the label functionality, because NewWithKubeConfig() doesn't allow one to set labels. If we don't make this change then users can either use RenewDeadline or LeaderLabels. This change lets them use both. If you'd prefer, I can cut a PR to client-go to create a NewWithKubeConfigAndLabels() function so we can keep both paths, but this change seems cleaner overall

// RenewDeadline to keep a single hung request from forcing a leader loss.
// Setting it to max(time.Second, RenewDeadline/2) as a reasonable heuristic.
if options.RenewDeadline != 0 {
return resourcelock.NewFromKubeconfig(options.LeaderElectionResourceLock,
options.LeaderElectionNamespace,
options.LeaderElectionID,
resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: recorderProvider.GetEventRecorderFor(id),
},
config,
options.RenewDeadline,
)
timeout := options.RenewDeadline / 2
if timeout < time.Second {
timeout = time.Second
}
config.Timeout = timeout
}

// Construct clients for leader election
corev1Client, err := corev1client.NewForConfig(config)
if err != nil {
return nil, err
Expand All @@ -118,7 +120,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
if err != nil {
return nil, err
}
return resourcelock.New(options.LeaderElectionResourceLock,

return resourcelock.NewWithLabels(options.LeaderElectionResourceLock,
options.LeaderElectionNamespace,
options.LeaderElectionID,
corev1Client,
Expand All @@ -127,6 +130,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
Identity: id,
EventRecorder: recorderProvider.GetEventRecorderFor(id),
},
options.LeaderLabels,
)
}

Expand Down
10 changes: 8 additions & 2 deletions pkg/manager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,15 @@ type Options struct {
// LeaseDuration time first.
LeaderElectionReleaseOnCancel bool

// LeaderElectionLabels allows a controller to supplement all leader election api calls with a set of custom labels based on
// the replica attempting to acquire leader status.
LeaderElectionLabels map[string]string

// LeaderElectionResourceLockInterface allows to provide a custom resourcelock.Interface that was created outside
// of the controller-runtime. If this value is set the options LeaderElectionID, LeaderElectionNamespace,
// LeaderElectionResourceLock, LeaseDuration, RenewDeadline and RetryPeriod will be ignored. This can be useful if you
// want to use a locking mechanism that is currently not supported, like a MultiLock across two Kubernetes clusters.
// LeaderElectionResourceLock, LeaseDuration, RenewDeadline, RetryPeriod and LeaderElectionLeases will be ignored.
// This can be useful if you want to use a locking mechanism that is currently not supported, like a MultiLock across
// two Kubernetes clusters.
LeaderElectionResourceLockInterface resourcelock.Interface

// LeaseDuration is the duration that non-leader candidates will
Expand Down Expand Up @@ -390,6 +395,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
LeaderElectionID: options.LeaderElectionID,
LeaderElectionNamespace: options.LeaderElectionNamespace,
RenewDeadline: *options.RenewDeadline,
LeaderLabels: options.LeaderElectionLabels,
})
if err != nil {
return nil, err
Expand Down
19 changes: 19 additions & 0 deletions pkg/manager/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,25 @@ var _ = Describe("manger.Manager", func() {
Expect(err).ToNot(HaveOccurred())
Expect(record.HolderIdentity).To(BeEmpty())
})
It("should set the leaselocks's label field when LeaderElectionLabels is set", func() {
labels := map[string]string{"my-key": "my-val"}
m, err := New(cfg, Options{
LeaderElection: true,
LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
LeaderElectionID: "controller-runtime",
LeaderElectionNamespace: "default",
LeaderElectionLabels: labels,
})
Expect(err).ToNot(HaveOccurred())
Expect(m).ToNot(BeNil())
cm, ok := m.(*controllerManager)
Expect(ok).To(BeTrue())
ll, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock)
Expect(isLeaseLock).To(BeTrue())
val, exists := ll.Labels["my-key"]
Expect(exists).To(BeTrue())
Expect(val).To(Equal("my-val"))
})
When("using a custom LeaderElectionResourceLockInterface", func() {
It("should use the custom LeaderElectionResourceLockInterface", func() {
rl, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{})
Expand Down