Skip to content

Commit 57f0058

Browse files
cnvergencezhaohuabing
cnvergence
and
zhaohuabing
authored
feat: validate JWT token and use projected token (#5871)
* Add proxyMetadata to xds config and validate JWT Signed-off-by: Karol Szwaj <[email protected]> * Add controller namespace to infra Signed-off-by: Karol Szwaj <[email protected]> * Add Metadata envoy bootstrap struct Signed-off-by: Karol Szwaj <[email protected]> * Add release note Signed-off-by: Karol Szwaj <[email protected]> * fix lint Signed-off-by: Karol Szwaj <[email protected]> * fix doc Signed-off-by: Karol Szwaj <[email protected]> * use projected service account tokens with eg audience Signed-off-by: Karol Szwaj <[email protected]> * lint code Signed-off-by: Karol Szwaj <[email protected]> * make gen Signed-off-by: Karol Szwaj <[email protected]> * make gen Signed-off-by: Karol Szwaj <[email protected]> * Revert "Add controller namespace to infra" This reverts commit b2fa2ca. Signed-off-by: Karol Szwaj <[email protected]> * fetch the node id and initial metadata from first msg Signed-off-by: Karol Szwaj <[email protected]> * update codegen Signed-off-by: Karol Szwaj <[email protected]> * verify service account Signed-off-by: Huabing (Robin) Zhao <[email protected]> * validate only sa Signed-off-by: Karol Szwaj <[email protected]> * add local hash name func Signed-off-by: Karol Szwaj <[email protected]> * Verify pod name for authz This reverts commit b0748a0. Signed-off-by: Huabing (Robin) Zhao <[email protected]> * lint code Signed-off-by: Karol Szwaj <[email protected]> --------- Signed-off-by: Karol Szwaj <[email protected]> Signed-off-by: Huabing (Robin) Zhao <[email protected]> Co-authored-by: Huabing (Robin) Zhao <[email protected]>
1 parent 9b78828 commit 57f0058

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+146
-94
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ require (
7171
k8s.io/api v0.33.0
7272
k8s.io/apiextensions-apiserver v0.33.0
7373
k8s.io/apimachinery v0.34.0-alpha.0
74+
k8s.io/apiserver v0.33.0
7475
k8s.io/cli-runtime v0.33.0
7576
k8s.io/client-go v0.33.0
7677
k8s.io/klog/v2 v2.130.1
@@ -497,7 +498,6 @@ require (
497498
gopkg.in/inf.v0 v0.9.1 // indirect
498499
gopkg.in/yaml.v2 v2.4.0 // indirect
499500
honnef.co/go/tools v0.6.1 // indirect
500-
k8s.io/apiserver v0.33.0 // indirect
501501
k8s.io/component-base v0.33.0 // indirect
502502
k8s.io/metrics v0.33.0 // indirect
503503
mvdan.cc/gofumpt v0.7.0 // indirect

‎internal/infrastructure/common/proxy_args.go

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func BuildProxyArgs(
3333
if bootstrapConfigOptions != nil && bootstrapConfigOptions.IPFamily == nil {
3434
bootstrapConfigOptions.IPFamily = getIPFamily(infra)
3535
}
36+
3637
bootstrapConfigOptions.GatewayNamespaceMode = gatewayNamespaceMode
3738
bootstrapConfigurations, err := bootstrap.GetRenderedBootstrapConfig(bootstrapConfigOptions)
3839
if err != nil {

‎internal/infrastructure/kubernetes/proxy/resource.go

+30-3
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
139139
Resources: *containerSpec.Resources,
140140
SecurityContext: expectedEnvoySecurityContext(containerSpec),
141141
Ports: ports,
142-
VolumeMounts: expectedContainerVolumeMounts(containerSpec),
142+
VolumeMounts: expectedContainerVolumeMounts(containerSpec, gatewayNamespaceMode),
143143
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
144144
TerminationMessagePath: "/dev/termination-log",
145145
StartupProbe: &corev1.Probe{
@@ -288,7 +288,7 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
288288
}
289289

290290
// expectedContainerVolumeMounts returns expected proxy container volume mounts.
291-
func expectedContainerVolumeMounts(containerSpec *egv1a1.KubernetesContainerSpec) []corev1.VolumeMount {
291+
func expectedContainerVolumeMounts(containerSpec *egv1a1.KubernetesContainerSpec, gatewayNamespaceMode bool) []corev1.VolumeMount {
292292
volumeMounts := []corev1.VolumeMount{
293293
{
294294
Name: "certs",
@@ -300,11 +300,19 @@ func expectedContainerVolumeMounts(containerSpec *egv1a1.KubernetesContainerSpec
300300
MountPath: "/sds",
301301
},
302302
}
303+
if gatewayNamespaceMode {
304+
volumeMounts = append(volumeMounts, corev1.VolumeMount{
305+
Name: "sa-token",
306+
MountPath: "/var/run/secrets/token",
307+
ReadOnly: true,
308+
})
309+
}
310+
303311
return resource.ExpectedContainerVolumeMounts(containerSpec, volumeMounts)
304312
}
305313

306314
// expectedVolumes returns expected proxy deployment volumes.
307-
func expectedVolumes(name string, gatewayNamespacedMode bool, pod *egv1a1.KubernetesPodSpec) []corev1.Volume {
315+
func expectedVolumes(name string, gatewayNamespacedMode bool, pod *egv1a1.KubernetesPodSpec, dnsDomain string) []corev1.Volume {
308316
var volumes []corev1.Volume
309317
certsVolume := corev1.Volume{
310318
Name: "certs",
@@ -335,6 +343,25 @@ func expectedVolumes(name string, gatewayNamespacedMode bool, pod *egv1a1.Kubern
335343
},
336344
},
337345
}
346+
saAudience := fmt.Sprintf("%s.%s.svc.%s", config.EnvoyGatewayServiceName, config.DefaultNamespace, dnsDomain)
347+
saTokenProjectedVolume := corev1.Volume{
348+
Name: "sa-token",
349+
VolumeSource: corev1.VolumeSource{
350+
Projected: &corev1.ProjectedVolumeSource{
351+
Sources: []corev1.VolumeProjection{
352+
{
353+
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
354+
Path: "sa-token",
355+
Audience: saAudience,
356+
ExpirationSeconds: ptr.To[int64](3600),
357+
},
358+
},
359+
},
360+
DefaultMode: ptr.To[int32](420),
361+
},
362+
},
363+
}
364+
volumes = append(volumes, saTokenProjectedVolume)
338365
}
339366

340367
volumes = append(volumes, certsVolume)

‎internal/infrastructure/kubernetes/proxy/resource_provider.go

+2-8
Original file line numberDiff line numberDiff line change
@@ -322,15 +322,14 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
322322
Containers: containers,
323323
InitContainers: deploymentConfig.InitContainers,
324324
ServiceAccountName: r.Name(),
325-
AutomountServiceAccountToken: expectedAutoMountServiceAccountToken(r.GatewayNamespaceMode),
326325
TerminationGracePeriodSeconds: expectedTerminationGracePeriodSeconds(proxyConfig.Spec.Shutdown),
327326
DNSPolicy: corev1.DNSClusterFirst,
328327
RestartPolicy: corev1.RestartPolicyAlways,
329328
SchedulerName: "default-scheduler",
330329
SecurityContext: deploymentConfig.Pod.SecurityContext,
331330
Affinity: deploymentConfig.Pod.Affinity,
332331
Tolerations: deploymentConfig.Pod.Tolerations,
333-
Volumes: expectedVolumes(r.infra.Name, r.GatewayNamespaceMode, deploymentConfig.Pod),
332+
Volumes: expectedVolumes(r.infra.Name, r.GatewayNamespaceMode, deploymentConfig.Pod, r.DNSDomain),
334333
ImagePullSecrets: deploymentConfig.Pod.ImagePullSecrets,
335334
NodeSelector: deploymentConfig.Pod.NodeSelector,
336335
TopologySpreadConstraints: deploymentConfig.Pod.TopologySpreadConstraints,
@@ -537,10 +536,6 @@ func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 {
537536
return ptr.To(int64(s))
538537
}
539538

540-
func expectedAutoMountServiceAccountToken(gatewayNamespacedMode bool) *bool {
541-
return ptr.To(gatewayNamespacedMode)
542-
}
543-
544539
func (r *ResourceRender) getPodSpec(
545540
containers, initContainers []corev1.Container,
546541
pod *egv1a1.KubernetesPodSpec,
@@ -550,15 +545,14 @@ func (r *ResourceRender) getPodSpec(
550545
Containers: containers,
551546
InitContainers: initContainers,
552547
ServiceAccountName: ExpectedResourceHashedName(r.infra.Name),
553-
AutomountServiceAccountToken: expectedAutoMountServiceAccountToken(r.GatewayNamespaceMode),
554548
TerminationGracePeriodSeconds: expectedTerminationGracePeriodSeconds(proxyConfig.Spec.Shutdown),
555549
DNSPolicy: corev1.DNSClusterFirst,
556550
RestartPolicy: corev1.RestartPolicyAlways,
557551
SchedulerName: "default-scheduler",
558552
SecurityContext: pod.SecurityContext,
559553
Affinity: pod.Affinity,
560554
Tolerations: pod.Tolerations,
561-
Volumes: expectedVolumes(r.infra.Name, r.GatewayNamespaceMode, pod),
555+
Volumes: expectedVolumes(r.infra.Name, r.GatewayNamespaceMode, pod, r.DNSDomain),
562556
ImagePullSecrets: pod.ImagePullSecrets,
563557
NodeSelector: pod.NodeSelector,
564558
TopologySpreadConstraints: pod.TopologySpreadConstraints,

‎internal/infrastructure/kubernetes/proxy/resource_provider_test.go

-1
Original file line numberDiff line numberDiff line change
@@ -1329,7 +1329,6 @@ func TestServiceAccount(t *testing.T) {
13291329
ns = tc.infra.GetProxyInfra().Namespace
13301330
}
13311331
r := NewResourceRender(ns, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
1332-
13331332
sa, err := r.ServiceAccount()
13341333
require.NoError(t, err)
13351334

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ spec:
3333
gateway.envoyproxy.io/owning-gateway-name: default
3434
gateway.envoyproxy.io/owning-gateway-namespace: default
3535
spec:
36-
automountServiceAccountToken: false
3736
containers:
3837
- args:
3938
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ spec:
2828
gateway.envoyproxy.io/owning-gateway-name: default
2929
gateway.envoyproxy.io/owning-gateway-namespace: default
3030
spec:
31-
automountServiceAccountToken: false
3231
containers:
3332
- args:
3433
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ spec:
4141
label1: value1-override
4242
label2: value2
4343
spec:
44-
automountServiceAccountToken: false
4544
containers:
4645
- args:
4746
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ spec:
3737
gateway.envoyproxy.io/owning-gateway-name: default
3838
gateway.envoyproxy.io/owning-gateway-namespace: default
3939
spec:
40-
automountServiceAccountToken: false
4140
containers:
4241
- args:
4342
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: false
4039
containers:
4140
- args:
4241
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: false
4039
containers:
4140
- args:
4241
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ spec:
3838
gateway.envoyproxy.io/owning-gateway-name: default
3939
gateway.envoyproxy.io/owning-gateway-namespace: default
4040
spec:
41-
automountServiceAccountToken: false
4241
containers:
4342
- args:
4443
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ spec:
3838
gateway.envoyproxy.io/owning-gateway-name: default
3939
gateway.envoyproxy.io/owning-gateway-namespace: default
4040
spec:
41-
automountServiceAccountToken: false
4241
containers:
4342
- args:
4443
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ spec:
3737
gateway.envoyproxy.io/owning-gateway-name: default
3838
gateway.envoyproxy.io/owning-gateway-namespace: default
3939
spec:
40-
automountServiceAccountToken: false
4140
containers:
4241
- args:
4342
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: false
4039
containers:
4140
- args:
4241
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ spec:
3232
gateway.envoyproxy.io/owning-gateway-name: default
3333
gateway.envoyproxy.io/owning-gateway-namespace: default
3434
spec:
35-
automountServiceAccountToken: false
3635
containers:
3736
- args:
3837
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/dual-stack.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: false
4039
containers:
4140
- args:
4241
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ spec:
3737
gateway.envoyproxy.io/owning-gateway-name: default
3838
gateway.envoyproxy.io/owning-gateway-namespace: default
3939
spec:
40-
automountServiceAccountToken: false
4140
containers:
4241
- args:
4342
- --service-cluster default

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/gateway-namespace-mode.yaml

+12-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: true
4039
containers:
4140
- args:
4241
- --service-cluster default
@@ -243,7 +242,7 @@ spec:
243242
- name: jwt-sa-bearer
244243
generic_secret:
245244
secret:
246-
filename: "/var/run/secrets/kubernetes.io/serviceaccount/token"
245+
filename: "/var/run/secrets/token/sa-token"
247246
overload_manager:
248247
refresh_interval: 0.25s
249248
resource_monitors:
@@ -336,6 +335,9 @@ spec:
336335
readOnly: true
337336
- mountPath: /sds
338337
name: sds
338+
- mountPath: /var/run/secrets/token
339+
name: sa-token
340+
readOnly: true
339341
- args:
340342
- envoy
341343
- shutdown-manager
@@ -414,6 +416,14 @@ spec:
414416
serviceAccountName: envoy-default-37a8eec1
415417
terminationGracePeriodSeconds: 360
416418
volumes:
419+
- name: sa-token
420+
projected:
421+
defaultMode: 420
422+
sources:
423+
- serviceAccountToken:
424+
audience: envoy-gateway.envoy-gateway-system.svc.cluster.local
425+
expirationSeconds: 3600
426+
path: sa-token
417427
- configMap:
418428
defaultMode: 420
419429
items:

‎internal/infrastructure/kubernetes/proxy/testdata/deployments/ipv6.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ spec:
3636
gateway.envoyproxy.io/owning-gateway-name: default
3737
gateway.envoyproxy.io/owning-gateway-namespace: default
3838
spec:
39-
automountServiceAccountToken: false
4039
containers:
4140
- args:
4241
- --service-cluster default

0 commit comments

Comments
 (0)