update grpc xds server and share eg ca cert in the infra configmap

Signed-off-by: Karol Szwaj <[email protected]>

Author: cnvergence

internal/infrastructure/kubernetes/infra.go

@@ -33,7 +33,7 @@ type ResourceRender interface {
 	LabelSelector() labels.Selector
 	ServiceAccount() (*corev1.ServiceAccount, error)
 	Service() (*corev1.Service, error)
-	ConfigMap() (*corev1.ConfigMap, error)
+	ConfigMap(cert string) (*corev1.ConfigMap, error)
 	Deployment() (*appsv1.Deployment, error)
 	DeploymentSpec() (*egv1a1.KubernetesDeploymentSpec, error)
 	DaemonSet() (*appsv1.DaemonSet, error)
@@ -80,11 +80,15 @@ func (i *Infra) Close() error { return nil }
 // createOrUpdate creates a ServiceAccount/ConfigMap/Deployment/Service in the kube api server based on the
 // provided ResourceRender, if it doesn't exist and updates it if it does.
 func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error {
+	cert, err := i.getEnvoyCA(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to fetch ca certificate for namespaced infra %s/%s: %w", i.Namespace, r.Name(), err)
+	}
 	if err := i.createOrUpdateServiceAccount(ctx, r); err != nil {
 		return fmt.Errorf("failed to create or update serviceaccount %s/%s: %w", i.Namespace, r.Name(), err)
 	}
 
-	if err := i.createOrUpdateConfigMap(ctx, r); err != nil {
+	if err := i.createOrUpdateConfigMap(ctx, r, cert); err != nil {
 		return fmt.Errorf("failed to create or update configmap %s/%s: %w", i.Namespace, r.Name(), err)
 	}
 

internal/infrastructure/kubernetes/infra_resource.go

@@ -24,6 +24,18 @@ import (
 	"github.com/envoyproxy/gateway/internal/metrics"
 )
 
+func (i *Infra) getEnvoyCA(ctx context.Context) (string, error) {
+	secret := &corev1.Secret{}
+	err := i.Client.Get(ctx, types.NamespacedName{
+		Name:      "envoy",
+		Namespace: "envoy-gateway-system",
+	}, secret)
+	if err != nil {
+		return "", err
+	}
+	return string(secret.Data["ca.crt"]), nil
+}
+
 // createOrUpdateServiceAccount creates a ServiceAccount in the kube api server based on the
 // provided ResourceRender, if it doesn't exist and updates it if it does.
 func (i *Infra) createOrUpdateServiceAccount(ctx context.Context, r ResourceRender) (err error) {
@@ -56,7 +68,7 @@ func (i *Infra) createOrUpdateServiceAccount(ctx context.Context, r ResourceRend
 
 // createOrUpdateConfigMap creates a ConfigMap in the Kube api server based on the provided
 // ResourceRender, if it doesn't exist and updates it if it does.
-func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender) (err error) {
+func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender, cert string) (err error) {
 	var (
 		cm        *corev1.ConfigMap
 		startTime = time.Now()
@@ -67,7 +79,7 @@ func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender) (
 		}
 	)
 
-	if cm, err = r.ConfigMap(); err != nil {
+	if cm, err = r.ConfigMap(cert); err != nil {
 		resourceApplyTotal.WithFailure(metrics.StatusFailure, labels...).Increment()
 		return err
 	}

internal/infrastructure/kubernetes/proxy/resource.go

@@ -82,7 +82,7 @@ func enablePrometheus(infra *ir.ProxyInfra) bool {
 func expectedProxyContainers(infra *ir.ProxyInfra,
 	containerSpec *egv1a1.KubernetesContainerSpec,
 	shutdownConfig *egv1a1.ShutdownConfig, shutdownManager *egv1a1.ShutdownManager,
-	namespace string, dnsDomain string, gatewayNamespaceMode bool,
+	egNamespace string, dnsDomain string, gatewayNamespaceMode bool,
 ) ([]corev1.Container, error) {
 	ports := make([]corev1.ContainerPort, 0, 2)
 	if enablePrometheus(infra) {
@@ -107,6 +107,9 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 
 	maxHeapSizeBytes := calculateMaxHeapSizeBytes(containerSpec.Resources)
 
+	if gatewayNamespaceMode {
+		egNamespace = config.DefaultNamespace
+	}
 	// Get the default Bootstrap
 	bootstrapConfigOptions := &bootstrap.RenderBootstrapConfigOptions{
 		ProxyMetrics: proxyMetrics,
@@ -115,7 +118,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 			TrustedCA:   filepath.Join("/sds", common.SdsCAFilename),
 		},
 		MaxHeapSizeBytes: maxHeapSizeBytes,
-		XdsServerHost:    ptr.To(fmt.Sprintf("%s.%s.svc.%s", config.EnvoyGatewayServiceName, namespace, dnsDomain)),
+		XdsServerHost:    ptr.To(fmt.Sprintf("%s.%s.svc.%s", config.EnvoyGatewayServiceName, egNamespace, dnsDomain)),
 	}
 
 	args, err := common.BuildProxyArgs(infra, shutdownConfig, bootstrapConfigOptions, fmt.Sprintf("$(%s)", envoyPodEnvVar), gatewayNamespaceMode)
@@ -130,7 +133,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 			ImagePullPolicy:          corev1.PullIfNotPresent,
 			Command:                  []string{"envoy"},
 			Args:                     args,
-			Env:                      expectedContainerEnv(containerSpec),
+			Env:                      expectedContainerEnv(containerSpec, egNamespace),
 			Resources:                *containerSpec.Resources,
 			SecurityContext:          expectedEnvoySecurityContext(containerSpec),
 			Ports:                    ports,
@@ -192,7 +195,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 			ImagePullPolicy:          corev1.PullIfNotPresent,
 			Command:                  []string{"envoy-gateway"},
 			Args:                     expectedShutdownManagerArgs(shutdownConfig),
-			Env:                      expectedContainerEnv(nil),
+			Env:                      expectedContainerEnv(nil, egNamespace),
 			Resources:                *egv1a1.DefaultShutdownManagerContainerResourceRequirements(),
 			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
 			TerminationMessagePath:   "/dev/termination-log",
@@ -285,14 +288,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
 // expectedContainerVolumeMounts returns expected proxy container volume mounts.
 func expectedContainerVolumeMounts(gatewayNamespacedMode bool, containerSpec *egv1a1.KubernetesContainerSpec) []corev1.VolumeMount {
 	var volumeMounts []corev1.VolumeMount
-	if !gatewayNamespacedMode {
-		certsMount := corev1.VolumeMount{
-			Name:      "certs",
-			MountPath: "/certs",
-			ReadOnly:  true,
-		}
-		volumeMounts = append(volumeMounts, certsMount)
+
+	certsMount := corev1.VolumeMount{
+		Name:      "certs",
+		MountPath: "/certs",
+		ReadOnly:  true,
 	}
+	volumeMounts = append(volumeMounts, certsMount)
+
 	sdsMount := corev1.VolumeMount{
 		Name:      "sds",
 		MountPath: "/sds",
@@ -305,20 +308,39 @@ func expectedContainerVolumeMounts(gatewayNamespacedMode bool, containerSpec *eg
 // expectedVolumes returns expected proxy deployment volumes.
 func expectedVolumes(name string, gatewayNamespacedMode bool, pod *egv1a1.KubernetesPodSpec) []corev1.Volume {
 	var volumes []corev1.Volume
+	certsVolume := corev1.Volume{
+		Name: "certs",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  "envoy",
+				DefaultMode: ptr.To[int32](420),
+			},
+		},
+	}
 
-	if !gatewayNamespacedMode {
-		certsVolume := corev1.Volume{
+	if gatewayNamespacedMode {
+		certsVolume = corev1.Volume{
 			Name: "certs",
 			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName:  "envoy",
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: ExpectedResourceHashedName(name),
+					},
+					Items: []corev1.KeyToPath{
+						{
+							Key:  XdsTLSCaFileName,
+							Path: XdsTLSCaFileName,
+						},
+					},
 					DefaultMode: ptr.To[int32](420),
+					Optional:    ptr.To(false),
 				},
 			},
 		}
-		volumes = append(volumes, certsVolume)
 	}
 
+	volumes = append(volumes, certsVolume)
+
 	sdsVolume := corev1.Volume{
 		Name: "sds",
 		VolumeSource: corev1.VolumeSource{
@@ -341,21 +363,36 @@ func expectedVolumes(name string, gatewayNamespacedMode bool, pod *egv1a1.Kubern
 			},
 		},
 	}
+	if gatewayNamespacedMode {
+		sdsVolume = corev1.Volume{
+			Name: "sds",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: ExpectedResourceHashedName(name),
+					},
+					Items: []corev1.KeyToPath{
+						{
+							Key:  common.SdsCAFilename,
+							Path: common.SdsCAFilename,
+						},
+					},
+					DefaultMode: ptr.To[int32](420),
+					Optional:    ptr.To(false),
+				},
+			},
+		}
+	}
 	volumes = append(volumes, sdsVolume)
 	return resource.ExpectedVolumes(pod, volumes)
 }
 
 // expectedContainerEnv returns expected proxy container envs.
-func expectedContainerEnv(containerSpec *egv1a1.KubernetesContainerSpec) []corev1.EnvVar {
+func expectedContainerEnv(containerSpec *egv1a1.KubernetesContainerSpec, egNamespace string) []corev1.EnvVar {
 	env := []corev1.EnvVar{
 		{
-			Name: envoyNsEnvVar,
-			ValueFrom: &corev1.EnvVarSource{
-				FieldRef: &corev1.ObjectFieldSelector{
-					APIVersion: "v1",
-					FieldPath:  "metadata.namespace",
-				},
-			},
+			Name:  envoyNsEnvVar,
+			Value: egNamespace,
 		},
 		{
 			Name: envoyPodEnvVar,

internal/infrastructure/kubernetes/proxy/resource_provider.go

@@ -37,6 +37,9 @@ const (
 	// XdsTLSCaFilepath is the fully qualified path of the file containing Envoy's
 	// trusted CA certificate.
 	XdsTLSCaFilepath = "/certs/ca.crt"
+
+	// XdsTLSCertFileName is the file name of the xDS server TLS certificate.
+	XdsTLSCaFileName = "ca.crt"
 )
 
 type ResourceRender struct {
@@ -216,7 +219,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) {
 }
 
 // ConfigMap returns the expected ConfigMap based on the provided infra.
-func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) {
+func (r *ResourceRender) ConfigMap(cert string) (*corev1.ConfigMap, error) {
 	// Set the labels based on the owning gateway name.
 	labels := envoyLabels(r.infra.GetProxyMetadata().Labels)
 	if OwningGatewayLabelsAbsent(labels) {
@@ -237,6 +240,7 @@ func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) {
 		Data: map[string]string{
 			common.SdsCAFilename:   common.GetSdsCAConfigMapData(XdsTLSCaFilepath),
 			common.SdsCertFilename: common.GetSdsCertConfigMapData(XdsTLSCertFilepath, XdsTLSKeyFilepath),
+			XdsTLSCaFileName:       cert,
 		},
 	}, nil
 }

internal/infrastructure/kubernetes/proxy/resource_provider_test.go

@@ -1243,7 +1243,7 @@ func TestConfigMap(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
-			cm, err := r.ConfigMap()
+			cm, err := r.ConfigMap("")
 			require.NoError(t, err)
 
 			expected, err := loadConfigmap(tc.name)

internal/infrastructure/kubernetes/proxy_configmap_test.go

@@ -112,7 +112,7 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) {
 			}
 			kube := NewInfra(cli, cfg)
 			r := proxy.NewResourceRender(kube.Namespace, kube.DNSDomain, infra.GetProxyInfra(), kube.EnvoyGateway)
-			err := kube.createOrUpdateConfigMap(context.Background(), r)
+			err := kube.createOrUpdateConfigMap(context.Background(), r, "")
 			require.NoError(t, err)
 			actual := &corev1.ConfigMap{
 				ObjectMeta: metav1.ObjectMeta{

internal/infrastructure/kubernetes/proxy_infra.go

@@ -37,6 +37,10 @@ func (i *Infra) DeleteProxyInfra(ctx context.Context, infra *ir.Infra) error {
 		return errors.New("infra ir is nil")
 	}
 
+	if i.EnvoyGateway.GatewayNamespaceMode() && i.Namespace == "" {
+		i.Namespace = infra.Proxy.Namespace
+	}
+
 	r := proxy.NewResourceRender(i.Namespace, i.DNSDomain, infra.GetProxyInfra(), i.EnvoyGateway)
 	return i.delete(ctx, r)
 }

internal/infrastructure/kubernetes/ratelimit/resource_provider.go

@@ -79,7 +79,7 @@ func enablePrometheus(rl *egv1a1.RateLimit) bool {
 }
 
 // ConfigMap returns the expected rate limit ConfigMap based on the provided infra.
-func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) {
+func (r *ResourceRender) ConfigMap(cert string) (*corev1.ConfigMap, error) {
 	if !enablePrometheus(r.rateLimit) {
 		return nil, nil
 	}

internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go

@@ -197,7 +197,7 @@ func TestConfigmap(t *testing.T) {
 		},
 	}
 	r := NewResourceRender(cfg.Namespace, cfg.EnvoyGateway, ownerReferenceUID)
-	cm, err := r.ConfigMap()
+	cm, err := r.ConfigMap("")
 	require.NoError(t, err)
 
 	if *overrideTestData {