feat: filtering logic for exluded paths (git file generator) must stay backwards-compatible with the default greedy git file generator gloobing (argoproj#13690)

Author: hubmat00

applicationset/generators/git.go

@@ -101,15 +101,17 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
 
 func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
 
-	// Get all files that match the requested path string, removing duplicates
+	// Get all files that match the requested path string but are not configured as excludes, removing duplicates
 	allFiles := make(map[string][]byte)
 	for _, requestedPath := range appSetGenerator.Git.Files {
-		files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
-		if err != nil {
-			return nil, err
-		}
-		for filePath, content := range files {
-			allFiles[filePath] = content
+		if !requestedPath.Exclude {
+			files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
+			if err != nil {
+				return nil, err
+			}
+			for filePath, content := range files {
+				allFiles[filePath] = content
+			}
 		}
 	}
 
@@ -121,7 +123,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	}
 	sort.Strings(allPaths)
 
-	filteredPaths := g.filterPaths(appSetGenerator.Git.Files, allPaths)
+	filteredPaths := g.filterFilePaths(appSetGenerator.Git.Files, allPaths)
 
 	// Generate params from each path, and return
 	res := []map[string]interface{}{}
@@ -243,24 +245,27 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitGeneratorI
 	return res
 }
 
-func (g *GitGenerator) filterPaths(items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
+func (g *GitGenerator) filterFilePaths(Files []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
 	res := []string{}
 	for _, itemPath := range allPaths {
 		include := false
 		exclude := false
-		for _, requestedPath := range items {
-			match, err := doublestar.Match(requestedPath.Path, itemPath)
-			if err != nil {
-				log.WithError(err).WithField("requestedPath", requestedPath).
-					WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
-				continue
-			}
-			if match && !requestedPath.Exclude {
+		for _, requestedPath := range Files {
+			// exec doublestar.Match only on excluded requestedPaths to stay backwards-compatible with the default
+			// greedy git file generator globbing
+			if requestedPath.Exclude {
+				match, err := doublestar.Match(requestedPath.Path, itemPath)
+				if err != nil {
+					log.WithError(err).WithField("requestedPath", requestedPath).
+						WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
+					continue
+				}
+				if match {
+					exclude = true
+				}
+			} else {
 				include = true
 			}
-			if match && requestedPath.Exclude {
-				exclude = true
-			}
 		}
 		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
 		if include && !exclude {

applicationset/generators/git_test.go

@@ -1087,6 +1087,51 @@ cluster:
 			},
 			expectedError: nil,
 		},
+		{
+			name:  "test compatibility with greedy git file generator globbing and exclude filter",
+			files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "some-path/*.yaml"}},
+			repoFileContents: map[string][]byte{
+				"some-path/values.yaml": []byte(`
+cluster:
+  owner: [email protected]
+  name: production
+  address: https://kubernetes.default.svc
+`),
+				"some-path/staging/values.yaml": []byte(`
+cluster:
+  owner: [email protected]
+  name: staging
+  address: https://kubernetes.default.svc
+`),
+			},
+			repoPathsError: nil,
+			expected: []map[string]interface{}{
+				{
+					"cluster.owner":           "[email protected]",
+					"cluster.name":            "production",
+					"cluster.address":         "https://kubernetes.default.svc",
+					"path":                    "some-path",
+					"path.basename":           "some-path",
+					"path[0]":                 "some-path",
+					"path.basenameNormalized": "some-path",
+					"path.filename":           "values.yaml",
+					"path.filenameNormalized": "values.yaml",
+				},
+				{
+					"cluster.owner":           "[email protected]",
+					"cluster.name":            "staging",
+					"cluster.address":         "https://kubernetes.default.svc",
+					"path":                    "some-path/staging",
+					"path.basename":           "staging",
+					"path[0]":                 "some-path",
+					"path[1]":                 "staging",
+					"path.basenameNormalized": "staging",
+					"path.filename":           "values.yaml",
+					"path.filenameNormalized": "values.yaml",
+				},
+			},
+			expectedError: nil,
+		},
 	}
 
 	for _, testCase := range cases {

docs/operator-manual/applicationset/Generators-Git.md

@@ -339,7 +339,7 @@ The filename can always be accessed using `{{path.filename}}`.
 
 ### Exclude files
 
-The Git file generator also supports an `exclude` option in order to exclude files in the repository from being scanned by the ApplicationSet controller:
+The Git file generator also supports an `exclude` option in order to exclude files/folders in the repository from being scanned by the ApplicationSet controller:
 
 ```yaml
 apiVersion: argoproj.io/v1alpha1
@@ -373,7 +373,7 @@ spec:
 ```
 (*The full example can be found [here](https://github.com/argoproj/argo-cd/tree/master/applicationset/examples/git-generator-files-discovery/excludes).*)
 
-This example excludes the config.json file in the `dev` directory from the list of files scanned for this `ApplictionSet` resource.
+This example excludes the config.json file in the `dev` directory from the list of files scanned for this `ApplicationSet` resource.
 
 File exclude paths are matched using [doublestar.Match](https://github.com/bmatcuk/doublestar/blob/master/match.go#L8)