feat: readyz check improvements

Author: mikouaj

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"net/http"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -136,13 +137,14 @@ func main() {
 	}
 
 	boostMgr := boost.NewManager(mgr.GetClient())
-	go setupControllers(mgr, boostMgr, cfg, podLevelResourcesEnabled, versionInfo, certsReady)
-
+	controllersReady := make(chan struct{})
+	go setupControllers(mgr, boostMgr, cfg, podLevelResourcesEnabled, versionInfo, certsReady,
+		controllersReady)
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+	if err := setupReadyzCheck(mgr, boostMgr, controllersReady); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
@@ -157,7 +159,9 @@ func main() {
 }
 
 func setupControllers(mgr ctrl.Manager, boostMgr boost.Manager, cfg *config.Config,
-	podLevelResourcesEnabled bool, serverVersion *version.Info, certsReady chan struct{}) {
+	podLevelResourcesEnabled bool, serverVersion *version.Info, certsReady chan struct{},
+	controllersReady chan struct{}) {
+	defer close(controllersReady)
 	setupLog.Info("Waiting for certificate generation to complete")
 	<-certsReady
 	setupLog.Info("Certificate generation has completed")
@@ -183,6 +187,29 @@ func setupControllers(mgr ctrl.Manager, boostMgr boost.Manager, cfg *config.Conf
 	//+kubebuilder:scaffold:builder
 }
 
+func setupReadyzCheck(mgr ctrl.Manager, boostMgr boost.Manager,
+	controllersReadyChan chan struct{}) error {
+	if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error {
+		controllersReady := false
+		select {
+		case <-controllersReadyChan:
+			controllersReady = true
+		default:
+		}
+		if !controllersReady {
+			return fmt.Errorf("controllers are not ready")
+		}
+		if !boostMgr.IsRunning(req.Context()) {
+			return fmt.Errorf("boost manager is not running")
+		}
+		return nil
+	}); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+	return nil
+}
+
 func getFeatureGates(clusterInfo util.ClusterInfo) (util.FeatureGates, error) {
 	setupLog.Info("fetching cluster feature gates")
 	return clusterInfo.GetFeatureGates()

internal/boost/manager.go

@@ -73,6 +73,9 @@ type Manager interface {
 
 	// Start starts the manager time based check loop.
 	Start(ctx context.Context) error
+
+	// IsRunning returns true if manager has started its time based check loop.
+	IsRunning(ctx context.Context) bool
 }
 
 type TimeTicker interface {
@@ -105,6 +108,7 @@ type podRevertTask struct {
 
 type managerImpl struct {
 	sync.RWMutex
+	isRunning     bool
 	client        client.Client
 	reconciler    reconcile.Reconciler
 	ticker        TimeTicker
@@ -240,7 +244,9 @@ func (m *managerImpl) SetStartupCPUBoostReconciler(reconciler reconcile.Reconcil
 // Start starts the manager time based check loop.
 func (m *managerImpl) Start(ctx context.Context) error {
 	defer m.ticker.Stop()
+	defer m.setRunning(false)
 	m.log.Info("starting")
+	m.setRunning(true)
 	for {
 		select {
 		case <-m.ticker.Tick():
@@ -252,8 +258,16 @@ func (m *managerImpl) Start(ctx context.Context) error {
 	}
 }
 
+func (m *managerImpl) IsRunning(ctx context.Context) bool {
+	return m.isRunning
+}
+
 // PRIVATE FUNCS START below
 
+func (m *managerImpl) setRunning(isRunning bool) {
+	m.isRunning = isRunning
+}
+
 // getMatchingBoost finds the most specific matching boost for a given pod.
 func (m *managerImpl) getMatchingBoost(pod *corev1.Pod) (StartupCPUBoost, bool) {
 	namespaceBoosts := m.regularBoosts.List(pod.Namespace)