feat: add the ability to exclude files when using the git file generator (argoproj#13690)

Signed-off-by: hubmat00 <[email protected]>

Author: hubmat00

applicationset/examples/git-generator-files-discovery/excludes/git-files-exclude-example.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: guestbook
+spec:
+  goTemplate: true
+  goTemplateOptions: ["missingkey=error"]
+  generators:
+    - git:
+        repoURL: https://github.com/argoproj/argo-cd.git
+        revision: HEAD
+        files:
+          - path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
+          - path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
+            exclude: true
+  template:
+    metadata:
+      name: '{{.cluster.name}}-guestbook'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/argo-cd.git
+        targetRevision: HEAD
+        path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
+      destination:
+        server: https://kubernetes.default.svc
+        #server: '{{.cluster.address}}'
+        namespace: guestbook

applicationset/examples/git-generator-files-discovery/excludes/git-generator-files-fasttemplate.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: guestbook
+spec:
+  generators:
+    - git:
+        repoURL: https://github.com/argoproj/argo-cd.git
+        revision: HEAD
+        files:
+        - path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
+        - path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
+          exclude: true
+  template:
+    metadata:
+      name: '{{cluster.name}}-guestbook'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/argo-cd.git
+        targetRevision: HEAD
+        path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
+      destination:
+        server: https://kubernetes.default.svc
+        #server: '{{cluster.address}}'
+        namespace: guestbook

applicationset/generators/generator_spec_processor_test.go

@@ -423,16 +423,16 @@ func TestInterpolateGenerator(t *testing.T) {
 	assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
 	assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])
 
-	fileNamePath := argov1alpha1.GitFileGeneratorItem{
+	fileNamePath := argov1alpha1.GitGeneratorItem{
 		Path: "{{name}}",
 	}
-	fileServerPath := argov1alpha1.GitFileGeneratorItem{
+	fileServerPath := argov1alpha1.GitGeneratorItem{
 		Path: "{{server}}",
 	}
 
 	requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
 		Git: &argov1alpha1.GitGenerator{
-			Files:    append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
+			Files:    append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
 			Template: argov1alpha1.ApplicationSetTemplate{},
 		},
 	}
@@ -477,16 +477,16 @@ func TestInterpolateGenerator_go(t *testing.T) {
 	assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
 	assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])
 
-	fileNamePath := argov1alpha1.GitFileGeneratorItem{
+	fileNamePath := argov1alpha1.GitGeneratorItem{
 		Path: "{{.name}}",
 	}
-	fileServerPath := argov1alpha1.GitFileGeneratorItem{
+	fileServerPath := argov1alpha1.GitGeneratorItem{
 		Path: "{{.server}}",
 	}
 
 	requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
 		Git: &argov1alpha1.GitGenerator{
-			Files:    append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
+			Files:    append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
 			Template: argov1alpha1.ApplicationSetTemplate{},
 		},
 	}
@@ -530,7 +530,7 @@ func TestInterpolateGeneratorError(t *testing.T) {
 		{name: "Error templating", args: args{
 			requestedGenerator: &argov1alpha1.ApplicationSetGenerator{Git: &argov1alpha1.GitGenerator{
 				RepoURL:  "foo",
-				Files:    []argov1alpha1.GitFileGeneratorItem{{Path: "bar/"}},
+				Files:    []argov1alpha1.GitGeneratorItem{{Path: "bar/"}},
 				Revision: "main",
 				Values: map[string]string{
 					"git_test":  "{{ toPrettyJson . }}",

applicationset/generators/git.go

@@ -3,6 +3,7 @@ package generators
 import (
 	"context"
 	"fmt"
+	"github.com/bmatcuk/doublestar/v4"
 	"path"
 	"sort"
 	"strconv"
@@ -120,9 +121,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	}
 	sort.Strings(allPaths)
 
+	filteredPaths := g.filterPaths(appSetGenerator.Git.Files, allPaths)
+
 	// Generate params from each path, and return
 	res := []map[string]interface{}{}
-	for _, path := range allPaths {
+	for _, path := range filteredPaths {
 
 		// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
 		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], appSetGenerator.Git.Values, useGoTemplate, goTemplateOptions, appSetGenerator.Git.PathParamPrefix)
@@ -212,7 +215,7 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 	return res, nil
 }
 
-func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
+func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
 	res := []string{}
 	for _, appPath := range allPaths {
 		appInclude := false
@@ -240,6 +243,33 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
 	return res
 }
 
+func (g *GitGenerator) filterPaths(items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
+	res := []string{}
+	for _, itemPath := range allPaths {
+		include := false
+		exclude := false
+		for _, requestedPath := range items {
+			match, err := doublestar.Match(requestedPath.Path, itemPath)
+			if err != nil {
+				log.WithError(err).WithField("requestedPath", requestedPath).
+					WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
+				continue
+			}
+			if match && !requestedPath.Exclude {
+				include = true
+			}
+			if match && requestedPath.Exclude {
+				exclude = true
+			}
+		}
+		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
+		if include && !exclude {
+			res = append(res, itemPath)
+		}
+	}
+	return res
+}
+
 func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
 	res := make([]map[string]interface{}, len(requestedApps))
 	for i, a := range requestedApps {