Skip to content

Commit f2b66f9

Browse files
hubmat00hubmat00
committed
feat: add the ability to exclude files when using the git file generator (argoproj#13690)
Signed-off-by: hubmat00 <[email protected]>
1 parent f8f9ae9 commit f2b66f9

21 files changed

+1513
-1362
lines changed

‎applicationset/examples/git-generator-files-discovery/excludes/git-files-exclude-example.yaml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: argoproj.io/v1alpha1
2+
kind: ApplicationSet
3+
metadata:
4+
name: guestbook
5+
spec:
6+
goTemplate: true
7+
goTemplateOptions: ["missingkey=error"]
8+
generators:
9+
- git:
10+
repoURL: https://github.com/argoproj/argo-cd.git
11+
revision: HEAD
12+
files:
13+
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
14+
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
15+
exclude: true
16+
template:
17+
metadata:
18+
name: '{{.cluster.name}}-guestbook'
19+
spec:
20+
project: default
21+
source:
22+
repoURL: https://github.com/argoproj/argo-cd.git
23+
targetRevision: HEAD
24+
path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
25+
destination:
26+
server: https://kubernetes.default.svc
27+
#server: '{{.cluster.address}}'
28+
namespace: guestbook

‎applicationset/examples/git-generator-files-discovery/excludes/git-generator-files-fasttemplate.yaml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: argoproj.io/v1alpha1
2+
kind: ApplicationSet
3+
metadata:
4+
name: guestbook
5+
spec:
6+
generators:
7+
- git:
8+
repoURL: https://github.com/argoproj/argo-cd.git
9+
revision: HEAD
10+
files:
11+
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
12+
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
13+
exclude: true
14+
template:
15+
metadata:
16+
name: '{{cluster.name}}-guestbook'
17+
spec:
18+
project: default
19+
source:
20+
repoURL: https://github.com/argoproj/argo-cd.git
21+
targetRevision: HEAD
22+
path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
23+
destination:
24+
server: https://kubernetes.default.svc
25+
#server: '{{cluster.address}}'
26+
namespace: guestbook

‎applicationset/generators/generator_spec_processor_test.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -423,16 +423,16 @@ func TestInterpolateGenerator(t *testing.T) {
423423
assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
424424
assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])
425425

426-
fileNamePath := argov1alpha1.GitFileGeneratorItem{
426+
fileNamePath := argov1alpha1.GitGeneratorItem{
427427
Path: "{{name}}",
428428
}
429-
fileServerPath := argov1alpha1.GitFileGeneratorItem{
429+
fileServerPath := argov1alpha1.GitGeneratorItem{
430430
Path: "{{server}}",
431431
}
432432

433433
requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
434434
Git: &argov1alpha1.GitGenerator{
435-
Files: append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
435+
Files: append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
436436
Template: argov1alpha1.ApplicationSetTemplate{},
437437
},
438438
}
@@ -477,16 +477,16 @@ func TestInterpolateGenerator_go(t *testing.T) {
477477
assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
478478
assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])
479479

480-
fileNamePath := argov1alpha1.GitFileGeneratorItem{
480+
fileNamePath := argov1alpha1.GitGeneratorItem{
481481
Path: "{{.name}}",
482482
}
483-
fileServerPath := argov1alpha1.GitFileGeneratorItem{
483+
fileServerPath := argov1alpha1.GitGeneratorItem{
484484
Path: "{{.server}}",
485485
}
486486

487487
requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
488488
Git: &argov1alpha1.GitGenerator{
489-
Files: append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
489+
Files: append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
490490
Template: argov1alpha1.ApplicationSetTemplate{},
491491
},
492492
}
@@ -530,7 +530,7 @@ func TestInterpolateGeneratorError(t *testing.T) {
530530
{name: "Error templating", args: args{
531531
requestedGenerator: &argov1alpha1.ApplicationSetGenerator{Git: &argov1alpha1.GitGenerator{
532532
RepoURL: "foo",
533-
Files: []argov1alpha1.GitFileGeneratorItem{{Path: "bar/"}},
533+
Files: []argov1alpha1.GitGeneratorItem{{Path: "bar/"}},
534534
Revision: "main",
535535
Values: map[string]string{
536536
"git_test": "{{ toPrettyJson . }}",

‎applicationset/generators/git.go

+44-9
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package generators
33
import (
44
"context"
55
"fmt"
6+
"github.com/bmatcuk/doublestar/v4"
67
"path"
78
"sort"
89
"strconv"
@@ -100,15 +101,17 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
100101

101102
func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
102103

103-
// Get all files that match the requested path string, removing duplicates
104+
// Get all files that match the requested path string but are not configured as excludes, removing duplicates
104105
allFiles := make(map[string][]byte)
105106
for _, requestedPath := range appSetGenerator.Git.Files {
106-
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
107-
if err != nil {
108-
return nil, err
109-
}
110-
for filePath, content := range files {
111-
allFiles[filePath] = content
107+
if !requestedPath.Exclude {
108+
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
109+
if err != nil {
110+
return nil, err
111+
}
112+
for filePath, content := range files {
113+
allFiles[filePath] = content
114+
}
112115
}
113116
}
114117

@@ -120,9 +123,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
120123
}
121124
sort.Strings(allPaths)
122125

126+
filteredPaths := g.filterFilePaths(appSetGenerator.Git.Files, allPaths)
127+
123128
// Generate params from each path, and return
124129
res := []map[string]interface{}{}
125-
for _, path := range allPaths {
130+
for _, path := range filteredPaths {
126131

127132
// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
128133
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], appSetGenerator.Git.Values, useGoTemplate, goTemplateOptions, appSetGenerator.Git.PathParamPrefix)
@@ -212,7 +217,7 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
212217
return res, nil
213218
}
214219

215-
func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
220+
func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
216221
res := []string{}
217222
for _, appPath := range allPaths {
218223
appInclude := false
@@ -240,6 +245,36 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
240245
return res
241246
}
242247

248+
func (g *GitGenerator) filterFilePaths(Files []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
249+
res := []string{}
250+
for _, itemPath := range allPaths {
251+
include := false
252+
exclude := false
253+
for _, requestedPath := range Files {
254+
// exec doublestar.Match only on excluded requestedPaths to stay backwards-compatible with the default
255+
// greedy git file generator globbing
256+
if requestedPath.Exclude {
257+
match, err := doublestar.Match(requestedPath.Path, itemPath)
258+
if err != nil {
259+
log.WithError(err).WithField("requestedPath", requestedPath).
260+
WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
261+
continue
262+
}
263+
if match {
264+
exclude = true
265+
}
266+
} else {
267+
include = true
268+
}
269+
}
270+
// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
271+
if include && !exclude {
272+
res = append(res, itemPath)
273+
}
274+
}
275+
return res
276+
}
277+
243278
func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
244279
res := make([]map[string]interface{}, len(requestedApps))
245280
for i, a := range requestedApps {

0 commit comments

Comments
 (0)