Switch most "git" operations over to https://github.com/src-d/go-git for a very dramatic speed increase (especially during dependency calculation)

Author: tianon

bashbrew/go/go.mod

@@ -4,7 +4,7 @@ require (
 	github.com/codegangsta/cli v1.20.0
 	github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1
 	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
-	golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
+	gopkg.in/src-d/go-git.v4 v4.11.0
 	pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041
 	pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a
 )

bashbrew/go/go.sum

@@ -1,16 +1,70 @@
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/codegangsta/cli v1.20.0 h1:iX1FXEgwzd5+XN6wk5cVHOGQj6Q3Dcp20lUeS4lHNTw=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
-github.com/docker-library/go-dockerlibrary v0.0.0-20190129000321-7e50189a05d4 h1:Jl830zF5XyeMipP9Ag2J64TAeheWDvOsqLuaDHn4pb8=
-github.com/docker-library/go-dockerlibrary v0.0.0-20190129000321-7e50189a05d4/go.mod h1:ijRhN3WM71dD8TfohKoUdX46BT2uz/Ek5O+5PINI880=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1 h1:eTBMV614FnFNrm4PKUB+8UtOyrcF1uINuBak//m6YI0=
 github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1/go.mod h1:ijRhN3WM71dD8TfohKoUdX46BT2uz/Ek5O+5PINI880=
+github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo=
+github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
+github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
+github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
+github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=
+gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
+gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs=
+gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git.v4 v4.11.0 h1:cJwWgJ0DXifrNrXM6RGN1Y2yR60Rr1zQ9Q5DX5S9qgU=
+gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041 h1:LmTwXQVDWXMigTB88hFtd6mc3l5eLAlgdtfPI0F69bQ=
 pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041/go.mod h1:e7Gva9AMoKtUKYJ1G9kIesbh+4VS2JnAOS8VWafyTCk=
 pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a h1:WwS7vlB5H2AtwKj1jsGwp2ZLud1x6WXRXh2fXsRqrcA=

bashbrew/go/src/bashbrew/docker.go

@@ -72,29 +72,24 @@ func (r Repo) archDockerfileMetadata(arch string, entry *manifest.Manifest2822En
 	if err != nil {
 		return nil, cli.NewMultiError(fmt.Errorf(`failed "git show" for %q from commit %q`, dockerfileFile, commit), err)
 	}
-	defer dockerfile.Close()
 
 	meta, err := parseDockerfileMetadata(dockerfile)
 	if err != nil {
 		return nil, cli.NewMultiError(fmt.Errorf(`failed parsing Dockerfile metadata for %q from commit %q`, dockerfileFile, commit), err)
 	}
 
-	if err := dockerfile.Close(); err != nil {
-		return nil, cli.NewMultiError(fmt.Errorf(`failed closing "git show" for %q from commit %q`, dockerfileFile, commit), err)
-	}
-
 	dockerfileMetadataCache[cacheKey] = meta
 	return meta, nil
 }
 
-func parseDockerfileMetadata(dockerfile io.Reader) (*dockerfileMetadata, error) {
+func parseDockerfileMetadata(dockerfile string) (*dockerfileMetadata, error) {
 	meta := &dockerfileMetadata{
 		// panic: assignment to entry in nil map
 		StageNameFroms: map[string]string{},
 		// (nil slices work fine)
 	}
 
-	scanner := bufio.NewScanner(dockerfile)
+	scanner := bufio.NewScanner(strings.NewReader(dockerfile))
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 

bashbrew/go/src/bashbrew/git.go

@@ -13,6 +13,10 @@ import (
 
 	"github.com/docker-library/go-dockerlibrary/manifest"
 	"github.com/docker-library/go-dockerlibrary/pkg/execpipe"
+
+	goGit "gopkg.in/src-d/go-git.v4"
+	goGitConfig "gopkg.in/src-d/go-git.v4/config"
+	goGitPlumbing "gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 func gitCache() string {
@@ -38,22 +42,54 @@ func git(args ...string) ([]byte, error) {
 	return out, err
 }
 
+var gitRepo *goGit.Repository
+
 func ensureGitInit() error {
-	err := os.MkdirAll(gitCache(), os.ModePerm)
+	if gitRepo != nil {
+		return nil
+	}
+
+	gitCacheDir := gitCache()
+	err := os.MkdirAll(gitCacheDir, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	gitRepo, err = goGit.PlainInit(gitCacheDir, true)
+	if err == goGit.ErrRepositoryAlreadyExists {
+		gitRepo, err = goGit.PlainOpen(gitCacheDir)
+	}
 	if err != nil {
 		return err
 	}
-	git("init", "--quiet", "--bare", ".") // ignore errors -- just make sure the repo exists
-	git("config", "gc.auto", "0")         // ensure garbage collection is disabled so we keep dangling commits
+
+	// ensure garbage collection is disabled so we keep dangling commits
+	config, err := gitRepo.Config()
+	if err != nil {
+		return err
+	}
+	config.Raw = config.Raw.SetOption("gc", "", "auto", "0")
+	gitRepo.Storer.SetConfig(config)
+
 	return nil
 }
 
+var fullGitCommitRegex = regexp.MustCompile(`^[0-9a-f]{40}$|^[0-9a-f]{64}$`)
+
 func getGitCommit(commit string) (string, error) {
-	out, err := git("rev-parse", commit+"^{commit}")
+	if fullGitCommitRegex.MatchString(commit) {
+		_, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
+		if err != nil {
+			return "", err
+		}
+		return commit, nil
+	}
+
+	h, err := gitRepo.ResolveRevision(goGitPlumbing.Revision(commit + "^{commit}"))
 	if err != nil {
 		return "", err
 	}
-	return strings.TrimSpace(string(out)), nil
+	return h.String(), nil
 }
 
 func gitStream(args ...string) (io.ReadCloser, error) {
@@ -69,8 +105,23 @@ func gitArchive(commit string, dir string) (io.ReadCloser, error) {
 	return gitStream("archive", "--format=tar", commit+":"+dir)
 }
 
-func gitShow(commit string, file string) (io.ReadCloser, error) {
-	return gitStream("show", commit+":"+path.Clean(file))
+func gitShow(commit string, file string) (string, error) {
+	gitCommit, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
+	if err != nil {
+		return "", err
+	}
+
+	gitFile, err := gitCommit.File(file)
+	if err != nil {
+		return "", err
+	}
+
+	contents, err := gitFile.Contents()
+	if err != nil {
+		return "", err
+	}
+
+	return contents, nil
 }
 
 // for gitNormalizeForTagUsage()
@@ -138,30 +189,50 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri
 			return commit, nil
 		}
 		fetchString += localRef
-	} else if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch {
-		// backwards compat (see manifest/line-based.go in go-dockerlibrary)
+	} else {
+		// we create a temporary remote dir so that we can clean it up completely afterwards
 		refBase := "refs/remotes"
 		refBaseDir := filepath.Join(gitCache(), refBase)
+
 		err := os.MkdirAll(refBaseDir, os.ModePerm)
 		if err != nil {
 			return "", err
 		}
+
 		tempRefDir, err := ioutil.TempDir(refBaseDir, "temp")
 		if err != nil {
 			return "", err
 		}
 		defer os.RemoveAll(tempRefDir)
+
 		tempRef := path.Join(refBase, filepath.Base(tempRefDir))
-		fetchString += tempRef + "/*"
-		// we create a temporary remote dir so that we can clean it up completely afterwards
+		if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch {
+			// backwards compat (see manifest/line-based.go in go-dockerlibrary)
+			fetchString += tempRef + "/*"
+		} else {
+			fetchString += tempRef + "/temp"
+		}
 	}
 
 	if strings.HasPrefix(entry.ArchGitRepo(arch), "git://github.com/") {
 		fmt.Fprintf(os.Stderr, "warning: insecure protocol git:// detected: %s\n", entry.ArchGitRepo(arch))
 		entry.SetGitRepo(arch, strings.Replace(entry.ArchGitRepo(arch), "git://", "https://", 1))
 	}
 
-	_, err = git("fetch", "--quiet", "--no-tags", entry.ArchGitRepo(arch), fetchString)
+	gitRemote, err := gitRepo.CreateRemoteAnonymous(&goGitConfig.RemoteConfig{
+		Name: "anonymous",
+		URLs: []string{entry.ArchGitRepo(arch)},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	err = gitRemote.Fetch(&goGit.FetchOptions{
+		RefSpecs: []goGitConfig.RefSpec{goGitConfig.RefSpec(fetchString)},
+		Tags:     goGit.NoTags,
+
+		//Progress: os.Stdout,
+	})
 	if err != nil {
 		return "", err
 	}
@@ -171,7 +242,7 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri
 		return "", err
 	}
 
-	_, err = git("tag", "--force", r.RepoName+"/"+entry.Tags[0], commit)
+	_, err = gitRepo.CreateTag(arch+"/"+r.RepoName+"/"+entry.Tags[0], goGitPlumbing.NewHash(commit), nil)
 	if err != nil {
 		return "", err
 	}