[tmpnet] Start kind cluster with golang (#3780)

Author: maru-ava

bin/tmpnetctl

@@ -10,4 +10,4 @@ cd "${AVALANCHE_PATH}"
 if [[ ! -f ./build/tmpnetctl ]]; then
    ./scripts/build_tmpnetctl.sh
 fi
-./build/tmpnetctl
+./build/tmpnetctl "${@}"

scripts/tests.e2e.bootstrap_monitor.sh

@@ -9,14 +9,6 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
   exit 255
 fi
 
-CMD=kind-with-registry.sh
-
-if ! command -v "${CMD}" &> /dev/null; then
-  echo "kind-with-registry.sh not found, have you run 'nix develop'?"
-  echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix"
-  exit 1
-fi
-
-"${CMD}"
+./bin/tmpnetctl start-kind-cluster
 
 KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e

tests/fixture/bootstrapmonitor/common.go

@@ -17,15 +17,13 @@ import (
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/version"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	restclient "k8s.io/client-go/rest"
 )
 
 // Path to write the details to on the data volume
@@ -223,25 +221,6 @@ func getLatestImageDetails(
 
 func getClientset(log logging.Logger) (*kubernetes.Clientset, error) {
 	log.Info("Initializing clientset")
-	kubeconfigPath := os.Getenv("KUBECONFIG")
-	var (
-		kubeconfig *restclient.Config
-		err        error
-	)
-	if len(kubeconfigPath) > 0 {
-		// Only use BuildConfigFromFlags if a path is provided to avoid the warning logs that
-		// will be omitted in a format that differs from the avalanchego format.
-		if kubeconfig, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath); err != nil {
-			return nil, fmt.Errorf("failed to build kubeconfig: %w", err)
-		}
-	} else {
-		if kubeconfig, err = restclient.InClusterConfig(); err != nil {
-			return nil, fmt.Errorf("failed to build kubeconfig: %w", err)
-		}
-	}
-	clientset, err := kubernetes.NewForConfig(kubeconfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create clientset: %w", err)
-	}
-	return clientset, nil
+	kubeConfigPath := os.Getenv("KUBECONFIG")
+	return tmpnet.GetClientset(log, kubeConfigPath, "")
 }

tests/fixture/bootstrapmonitor/e2e/e2e_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/ids"
@@ -105,7 +104,7 @@ var _ = ginkgo.Describe("[Bootstrap Tester]", func() {
 
 		ginkgo.By("Configuring a kubernetes client")
 		kubeconfigPath := os.Getenv("KUBECONFIG")
-		kubeconfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+		kubeconfig, err := tmpnet.GetClientConfig(tc.Log(), kubeconfigPath, "")
 		require.NoError(err)
 		clientset, err := kubernetes.NewForConfig(kubeconfig)
 		require.NoError(err)

tests/fixture/tmpnet/README.md

@@ -41,6 +41,7 @@ the following non-test files:
 | node.go                     | Node        | Orchestrates and configures nodes                           |
 | node_config.go              | Node        | Reads and writes node configuration                         |
 | node_process.go             | NodeProcess | Orchestrates node processes                                 |
+| start_kind_cluster.go       |             | Starts a local kind cluster                                 |
 | subnet.go                   | Subnet      | Orchestrates subnets                                        |
 | utils.go                    |             | Defines shared utility functions                            |
 

tests/fixture/tmpnet/cmd/main.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"go.uber.org/zap"
 
 	"github.com/ava-labs/avalanchego/tests"
@@ -236,9 +237,44 @@ func main() {
 	)
 	rootCmd.AddCommand(checkLogsCmd)
 
+	var (
+		kubeConfigPath    string
+		kubeConfigContext string
+	)
+	startKindClusterCmd := &cobra.Command{
+		Use:   "start-kind-cluster",
+		Short: "Starts a local kind cluster with an integrated registry",
+		RunE: func(*cobra.Command, []string) error {
+			ctx, cancel := context.WithTimeout(context.Background(), tmpnet.DefaultNetworkTimeout)
+			defer cancel()
+			log, err := tests.LoggerForFormat("", rawLogFormat)
+			if err != nil {
+				return err
+			}
+			return tmpnet.StartKindCluster(ctx, log, kubeConfigPath, kubeConfigContext)
+		},
+	}
+	SetKubeConfigFlags(startKindClusterCmd.PersistentFlags(), &kubeConfigPath, &kubeConfigContext)
+	rootCmd.AddCommand(startKindClusterCmd)
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "tmpnetctl failed: %v\n", err)
 		os.Exit(1)
 	}
 	os.Exit(0)
 }
+
+func SetKubeConfigFlags(flagSet *pflag.FlagSet, kubeConfigPath *string, kubeConfigContext *string) {
+	flagSet.StringVar(
+		kubeConfigPath,
+		"kubeconfig",
+		os.Getenv("KUBECONFIG"),
+		"The path to a kubernetes configuration file for the target cluster",
+	)
+	flagSet.StringVar(
+		kubeConfigContext,
+		"kubeconfig-context",
+		"",
+		"The path to a kubernetes configuration file for the target cluster",
+	)
+}

tests/fixture/tmpnet/kube.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/portforward"
 	"k8s.io/client-go/transport/spdy"
 	"k8s.io/utils/pointer"
@@ -299,3 +300,43 @@ func enableLocalForwardForPod(
 	}
 	return forwardedPorts[0].Local, stopChan, nil
 }
+
+// GetClientConfig replicates the behavior of clientcmd.BuildConfigFromFlags with zap logging and
+// support for an optional config context. If path is not provided, use of in-cluster config will
+// be attempted.
+func GetClientConfig(log logging.Logger, path string, context string) (*restclient.Config, error) {
+	if len(path) == 0 {
+		log.Warn("--kubeconfig not set.  Using the inClusterConfig.  This might not work.")
+		kubeconfig, err := restclient.InClusterConfig()
+		if err == nil {
+			return kubeconfig, nil
+		}
+		log.Warn("failed to create inClusterConfig, falling back to default config",
+			zap.Error(err),
+		)
+	}
+	overrides := &clientcmd.ConfigOverrides{}
+	if len(context) > 0 {
+		overrides.CurrentContext = context
+	}
+	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{
+			ExplicitPath: path,
+		},
+		overrides,
+	).ClientConfig()
+}
+
+// GetClientset returns a kubernetes clientset for the provided kubeconfig path and context.
+func GetClientset(log logging.Logger, path string, context string) (*kubernetes.Clientset, error) {
+	clientConfig, err := GetClientConfig(log, path, context)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get client config: %w", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create clientset: %w", err)
+	}
+	return clientset, nil
+}

tests/fixture/tmpnet/start_kind_cluster.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package tmpnet
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"go.uber.org/zap"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+// CheckClusterRunning checks if the configured cluster is accessible.
+// TODO(marun) Maybe differentiate between configuration and endpoint errors
+func CheckClusterRunning(log logging.Logger, configPath string, configContext string) error {
+	clientset, err := GetClientset(log, configPath, configContext)
+	if err != nil {
+		return err
+	}
+	// Check if the configured context can reach a cluster endpoint
+	_, err = clientset.Discovery().ServerVersion()
+	return err
+}
+
+// StartKindCluster starts a new kind cluster if one is not already running.
+func StartKindCluster(ctx context.Context, log logging.Logger, configPath string, configContext string) error {
+	err := CheckClusterRunning(log, configPath, configContext)
+	if err == nil {
+		log.Info("kubernetes cluster already running",
+			zap.String("kubeconfig", configPath),
+			zap.String("kubeconfigContext", configContext),
+		)
+		return nil
+	}
+
+	log.Debug("kubernetes cluster not running",
+		zap.String("kubeconfig", configPath),
+		zap.String("kubeconfigContext", configContext),
+		zap.Error(err),
+	)
+
+	// Start a new kind cluster
+	ctx, cancel := context.WithTimeout(ctx, DefaultNetworkTimeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "kind-with-registry.sh")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run kind-with-registry.sh: %w", err)
+	}
+	return nil
+}