Fixes for secrets and mounts

- Add new mount for `scdfmetadata` which gets creds for docker
  metadata fetching.
- Move all mounts under /workspace as this helps avoid issues
  mounting under /etc if no root is available.
- Also remove maven settings.
- Fixes #4776
- Fixes #4775
- Fixes #4774

Author: jvalkeal

src/carvel/config/dataflow-deployment.yml

@@ -3,6 +3,7 @@
 #@ load("dataflow.star", "ctr_image")
 #@ load("dataflow.star", "dataflow_container_env")
 #@ load("dataflow.star", "has_image_pull_secrets")
+#@ load("dataflow.star", "registry_secret_ref")
 #@ load("dataflow.star", "image_pull_secrets")
 #@ load("dataflow.star", "dataflow_liveness_path")
 #@ load("dataflow.star", "dataflow_readiness_path")
@@ -35,7 +36,10 @@ spec:
           mountPath: /workspace/config
           readOnly: true
         - name: database
-          mountPath: /etc/secrets/database
+          mountPath: /workspace/runtime/secrets/database
+          readOnly: true
+        - name: scdfmetadata
+          mountPath: /workspace/runtime/secrets
           readOnly: true
         ports:
         - containerPort: 80
@@ -73,3 +77,12 @@ spec:
       - name: database
         secret:
           secretName: db-dataflow
+      #@ if has_image_pull_secrets():
+      - name: scdfmetadata
+        secret:
+          secretName: #@ registry_secret_ref()
+      #@ else:
+      - name: scdfmetadata
+        secret:
+          secretName: reg-creds
+      #@ end

src/carvel/config/dataflow.star

@@ -32,10 +32,9 @@ def dataflow_container_env():
   envs.extend([{"name": "SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI", "value": "docker://" + ctr_image()}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_CONFIG_ENABLE_API", "value": "false"}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API", "value": "false"}])
-  envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_PATHS", "value": "/etc/secrets"}])
+  envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_PATHS", "value": "/workspace/runtime/secrets"}])
   envs.extend([{"name": "SPRING_CLOUD_DATAFLOW_SERVER_URI", "value": "http://${SCDF_SERVER_SERVICE_HOST}:${SCDF_SERVER_SERVICE_PORT}"}])
   envs.extend([{"name": "SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI", "value": "http://${SKIPPER_SERVICE_HOST}:${SKIPPER_SERVICE_PORT}/api"}])
-  envs.extend([{"name": "SPRING_APPLICATION_JSON", "value": "{ \"maven\": { \"local-repository\": null, \"remote-repositories\": { \"repo1\": { \"url\": \"https://repo.spring.io/libs-snapshot\"} } } }"}])
   if grafana_enabled():
     envs.extend([{"name": "MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED", "value": "true"}])
     envs.extend([{"name": "MANAGEMENT_METRICS_EXPORT_PROMETHEUS_RSOCKET_ENABLED", "value": "true"}])

src/carvel/config/skipper-deployment.yml

@@ -33,11 +33,11 @@ spec:
           mountPath: /workspace/config
           readOnly: true
         - name: database
-          mountPath: /etc/secrets/database
+          mountPath: /workspace/runtime/secrets/database
           readOnly: true
         #@ if rabbitmq_enabled():
         - name: rabbitmq
-          mountPath: /etc/secrets/rabbitmq
+          mountPath: /workspace/runtime/secrets/rabbitmq
           readOnly: true
         #@ end
         ports:

src/carvel/config/skipper.star

@@ -41,7 +41,7 @@ def skipper_container_env():
   envs.extend([{"name": "SPRING_CLOUD_CONFIG_ENABLED", "value": "false"}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_CONFIG_ENABLE_API", "value": "false"}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API", "value": "false"}])
-  envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_PATHS", "value": "/etc/secrets"}])
+  envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_PATHS", "value": "/workspace/runtime/secrets"}])
   if grafana_enabled():
     envs.extend([{"name": "MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED", "value": "true"}])
     envs.extend([{"name": "MANAGEMENT_METRICS_EXPORT_PROMETHEUS_RSOCKET_ENABLED", "value": "true"}])

src/carvel/test/binders.test.ts

@@ -53,7 +53,7 @@ describe('binders rabbit', () => {
     expect(rabbitVolume?.secret?.secretName).toBe('rabbitmq');
 
     const rabbitVolumeMount = containerVolumeMount(skipperContainer, 'rabbitmq');
-    expect(rabbitVolumeMount?.mountPath).toBe('/etc/secrets/rabbitmq');
+    expect(rabbitVolumeMount?.mountPath).toBe('/workspace/runtime/secrets/rabbitmq');
   });
 
   it('should skip deploy if external settings', async () => {

src/carvel/test/secrets.test.ts

@@ -1,6 +1,7 @@
 import 'jest-extended';
-import { DEFAULT_REQUIRED_DATA_VALUES } from '../src/constants';
-import { findPodSpecsWithImagePullSecrets, findSecret } from '../src/k8s-helper';
+import { SCDF_SERVER_NAME, DEFAULT_REQUIRED_DATA_VALUES } from '../src/constants';
+import { findDeployment, deploymentContainer, findPodSpecsWithImagePullSecrets, findSecret } from '../src/k8s-helper';
+
 import { execYtt } from '../src/ytt';
 
 describe('secrets', () => {
@@ -84,6 +85,22 @@ describe('secrets', () => {
     expect(secret).toBeTruthy();
   });
 
+  it('should add carvel secretgen on default 4', async () => {
+    const result = await execYtt({
+      files: ['config'],
+      dataValueYamls: [...DEFAULT_REQUIRED_DATA_VALUES]
+    });
+    expect(result.success, result.stderr).toBeTruthy();
+    const yaml = result.stdout;
+
+    const dataflowDeployment = findDeployment(yaml, SCDF_SERVER_NAME);
+    const dataflowContainer = deploymentContainer(dataflowDeployment, SCDF_SERVER_NAME);
+    const volumeMount = dataflowContainer?.volumeMounts?.find(x => x.name === 'scdfmetadata');
+    expect(volumeMount?.mountPath).toBe('/workspace/runtime/secrets');
+    const volume = dataflowDeployment?.spec?.template.spec?.volumes?.find(x => x.name === 'scdfmetadata');
+    expect(volume?.secret?.secretName).toBe('reg-creds');
+  });
+
   it('should add manual image pull secret if defined 1', async () => {
     const result = await execYtt({
       files: ['config'],
@@ -155,4 +172,20 @@ describe('secrets', () => {
     const secret = findSecret(yaml, 'reg-creds');
     expect(secret).toBeFalsy();
   });
+
+  it('should add manual image pull secret if defined 4', async () => {
+    const result = await execYtt({
+      files: ['config'],
+      dataValueYamls: [...DEFAULT_REQUIRED_DATA_VALUES, 'scdf.registry.secret.ref=fakeref']
+    });
+    expect(result.success, result.stderr).toBeTruthy();
+    const yaml = result.stdout;
+
+    const dataflowDeployment = findDeployment(yaml, SCDF_SERVER_NAME);
+    const dataflowContainer = deploymentContainer(dataflowDeployment, SCDF_SERVER_NAME);
+    const volumeMount = dataflowContainer?.volumeMounts?.find(x => x.name === 'scdfmetadata');
+    expect(volumeMount?.mountPath).toBe('/workspace/runtime/secrets');
+    const volume = dataflowDeployment?.spec?.template.spec?.volumes?.find(x => x.name === 'scdfmetadata');
+    expect(volume?.secret?.secretName).toBe('fakeref');
+  });
 });

src/carvel/test/servers.test.ts

@@ -268,7 +268,7 @@ describe('servers', () => {
         }),
         expect.objectContaining({
           name: 'SPRING_CLOUD_KUBERNETES_SECRETS_PATHS',
-          value: '/etc/secrets'
+          value: '/workspace/runtime/secrets'
         })
       ])
     );
@@ -374,7 +374,7 @@ describe('servers', () => {
         }),
         expect.objectContaining({
           name: 'SPRING_CLOUD_KUBERNETES_SECRETS_PATHS',
-          value: '/etc/secrets'
+          value: '/workspace/runtime/secrets'
         }),
         expect.objectContaining({
           name: 'SPRING_CLOUD_DATAFLOW_SERVER_URI'