support ignoring file or directories

use the doublestar library to support pattern matching of files or
directories to ignore. This replaces (and deprecates) the previous
-skip flag which only supported file extensions.

Author: willnorris

.github/workflows/tests.yaml

@@ -5,7 +5,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.x, 1.15.x]
+        go-version: [1.x, 1.16.x]
         platform: [ubuntu-latest]
         include:
           # only update test coverage stats with the most recent go version on linux

README.md

@@ -19,10 +19,14 @@ to any file that already has one.
     -l license type: apache, bsd, mit, mpl (defaults to "apache")
     -y year (defaults to current year)
     -check check only mode: verify presence of license headers and exit with non-zero code if missing
+    -ignore file patterns to ignore, for example: -ignore **/*.go -ignore vendor/**
 
 The pattern argument can be provided multiple times, and may also refer
 to single files.
 
+The `-ignore` flag can use any pattern [supported by
+doublestar](https://github.com/bmatcuk/doublestar#patterns).
+
 ## Running in a Docker Container
 
 - Clone the repository using `git clone https://github.com/google/addlicense.git`

go.mod

@@ -2,4 +2,7 @@ module github.com/google/addlicense
 
 go 1.13
 
-require golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+require (
+	github.com/bmatcuk/doublestar/v4 v4.0.2
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+)

go.sum

@@ -1,2 +1,4 @@
+github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
+github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

main.go

@@ -30,6 +30,7 @@ import (
 	"text/template"
 	"time"
 
+	doublestar "github.com/bmatcuk/doublestar/v4"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -48,7 +49,8 @@ Flags:
 `
 
 var (
-	skipExtensionFlags skipExtensionFlag
+	skipExtensionFlags stringSlice
+	ignorePatterns     stringSlice
 	spdx               spdxFlag
 
 	holder    = flag.String("c", "Google LLC", "copyright holder")
@@ -64,17 +66,19 @@ func init() {
 		fmt.Fprintln(os.Stderr, helpText)
 		flag.PrintDefaults()
 	}
-	flag.Var(&skipExtensionFlags, "skip", "To skip files to check/add the header file, for example: -skip rb -skip go")
+	flag.Var(&skipExtensionFlags, "skip", "[deprecated: see -ignore] file extensions to skip, for example: -skip rb -skip go")
+	flag.Var(&ignorePatterns, "ignore", "file patterns to ignore, for example: -ignore **/*.go -ignore vendor/**")
 	flag.Var(&spdx, "s", "Include SPDX identifier in license header. Set -s=only to only include SPDX identifier.")
 }
 
-type skipExtensionFlag []string
+// stringSlice stores the results of a repeated command line flag as a string slice.
+type stringSlice []string
 
-func (i *skipExtensionFlag) String() string {
+func (i *stringSlice) String() string {
 	return fmt.Sprint(*i)
 }
 
-func (i *skipExtensionFlag) Set(value string) error {
+func (i *stringSlice) Set(value string) error {
 	*i = append(*i, value)
 	return nil
 }
@@ -109,6 +113,17 @@ func main() {
 		os.Exit(1)
 	}
 
+	// convert -skip flags to -ignore equivalents
+	for _, s := range skipExtensionFlags {
+		ignorePatterns = append(ignorePatterns, fmt.Sprintf("**/*.%s", s))
+	}
+	// verify that all ignorePatterns are valid
+	for _, p := range ignorePatterns {
+		if !doublestar.ValidatePattern(p) {
+			log.Fatalf("-ignore pattern %q is not valid", p)
+		}
+	}
+
 	// map legacy license values
 	if t, ok := legacyLicenseTypes[*license]; ok {
 		*license = t
@@ -200,17 +215,27 @@ func walk(ch chan<- *file, start string) error {
 		if fi.IsDir() {
 			return nil
 		}
-		for _, skip := range skipExtensionFlags {
-			if strings.TrimPrefix(filepath.Ext(fi.Name()), ".") == skip || fi.Name() == skip {
-				log.Printf("%s: skipping this file", fi.Name())
-				return nil
-			}
+		if fileMatches(path, ignorePatterns) {
+			log.Printf("skipping: %s", path)
+			return nil
 		}
 		ch <- &file{path, fi.Mode()}
 		return nil
 	})
 }
 
+// fileMatches determines if path matches one of the provided file patterns.
+// Patterns are assumed to be valid.
+func fileMatches(path string, patterns []string) bool {
+	for _, p := range patterns {
+		// ignore error, since we assume patterns are valid
+		if match, _ := doublestar.Match(p, path); match {
+			return true
+		}
+	}
+	return false
+}
+
 // addLicense add a license to the file if missing.
 //
 // It returns true if the file was updated.

main_test.go

@@ -397,3 +397,63 @@ func TestHasLicense(t *testing.T) {
 		}
 	}
 }
+
+func TestFileMatches(t *testing.T) {
+	tests := []struct {
+		pattern   string
+		path      string
+		wantMatch bool
+	}{
+		// basic single directory patterns
+		{"", "file.c", false},
+		{"*.c", "file.h", false},
+		{"*.c", "file.c", true},
+
+		// subdirectory patterns
+		{"*.c", "vendor/file.c", false},
+		{"**/*.c", "vendor/file.c", true},
+		{"vendor/**", "vendor/file.c", true},
+		{"vendor/**/*.c", "vendor/file.c", true},
+		{"vendor/**/*.c", "vendor/a/b/file.c", true},
+
+		// single character "?" match
+		{"*.?", "file.c", true},
+		{"*.?", "file.go", false},
+		{"*.??", "file.c", false},
+		{"*.??", "file.go", true},
+
+		// character classes - sets and ranges
+		{"*.[ch]", "file.c", true},
+		{"*.[ch]", "file.h", true},
+		{"*.[ch]", "file.ch", false},
+		{"*.[a-z]", "file.c", true},
+		{"*.[a-z]", "file.h", true},
+		{"*.[a-z]", "file.go", false},
+		{"*.[a-z]", "file.R", false},
+
+		// character classes - negations
+		{"*.[^ch]", "file.c", false},
+		{"*.[^ch]", "file.h", false},
+		{"*.[^ch]", "file.R", true},
+		{"*.[!ch]", "file.c", false},
+		{"*.[!ch]", "file.h", false},
+		{"*.[!ch]", "file.R", true},
+
+		// comma-separated alternative matches
+		{"*.{c,go}", "file.c", true},
+		{"*.{c,go}", "file.go", true},
+		{"*.{c,go}", "file.h", false},
+
+		// negating alternative matches
+		{"*.[^{c,go}]", "file.c", false},
+		{"*.[^{c,go}]", "file.go", false},
+		{"*.[^{c,go}]", "file.h", true},
+	}
+
+	for _, tt := range tests {
+		patterns := []string{tt.pattern}
+		if got := fileMatches(tt.path, patterns); got != tt.wantMatch {
+			t.Errorf("fileMatches(%q, %q) returned %v, want %v", tt.path, patterns, got, tt.wantMatch)
+		}
+	}
+}