Refactor caching

pkg/skaffold/build/cache/cache.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2019 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"context"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
+	skafconfig "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
+	"github.com/docker/docker/api/types"
+
+	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/config"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// ArtifactCache is a map of [artifact dependencies hash : ImageDetails]
+type ArtifactCache map[string]ImageDetails
+
+// Cache holds any data necessary for accessing the cache
+type Cache struct {
+	artifactCache ArtifactCache
+	client        docker.LocalDaemon
+	builder       build.Builder
+	imageList     []types.ImageSummary
+	cacheFile     string
+	useCache      bool
+	needsPush     bool
+	localCluster  bool
+}
+
+var (
+	// For testing
+	localCluster    = config.GetLocalCluster
+	remoteDigest    = docker.RemoteDigest
+	newDockerCilent = docker.NewAPIClient
+	noCache         = &Cache{}
+)
+
+// NewCache returns the current state of the cache
+func NewCache(ctx context.Context, builder build.Builder, opts *skafconfig.SkaffoldOptions, needsPush bool) *Cache {
+	if !opts.CacheArtifacts {
+		return noCache
+	}
+	cf, err := resolveCacheFile(opts.CacheFile)
+	if err != nil {
+		logrus.Warnf("Error resolving cache file, not using skaffold cache: %v", err)
+		return noCache
+	}
+	cache, err := retrieveArtifactCache(cf)
+	if err != nil {
+		logrus.Warnf("Error retrieving artifact cache, not using skaffold cache: %v", err)
+		return noCache
+	}
+	client, err := newDockerCilent()
+	if err != nil {
+		logrus.Warnf("Error retrieving local daemon client; local daemon will not be used as a cache: %v", err)
+	}
+	var imageList []types.ImageSummary
+	if client != nil {
+		imageList, err = client.ImageList(ctx, types.ImageListOptions{})
+		if err != nil {
+			logrus.Warn("Unable to get list of images from local docker daemon, won't be checked for cache.")
+		}
+	}
+
+	lc, err := localCluster()
+	if err != nil {
+		logrus.Warn("Unable to determine if using a local cluster, cache may not work.")
+	}
+	return &Cache{
+		artifactCache: cache,
+		cacheFile:     cf,
+		useCache:      opts.CacheArtifacts,
+		client:        client,
+		builder:       builder,
+		needsPush:     needsPush,
+		imageList:     imageList,
+		localCluster:  lc,
+	}
+}
+
+// resolveCacheFile makes sure that either a passed in cache file or the default cache file exists
+func resolveCacheFile(cacheFile string) (string, error) {
+	if cacheFile != "" {
+		return cacheFile, util.VerifyOrCreateFile(cacheFile)
+	}
+	home, err := homedir.Dir()
+	if err != nil {
+		return "", errors.Wrap(err, "retrieving home directory")
+	}
+	defaultFile := filepath.Join(home, constants.DefaultSkaffoldDir, constants.DefaultCacheFile)
+	return defaultFile, util.VerifyOrCreateFile(defaultFile)
+}
+
+func retrieveArtifactCache(cacheFile string) (ArtifactCache, error) {
+	cache := ArtifactCache{}
+	contents, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		return nil, err
+	}
+	if err := yaml.Unmarshal(contents, &cache); err != nil {
+		return nil, err
+	}
+	return cache, nil
+}

pkg/skaffold/build/cache/cache_test.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2019 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/testutil"
+	"github.com/docker/docker/api/types"
+	yaml "gopkg.in/yaml.v2"
+)
+
+var (
+	digest    = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+	digestOne = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	image     = fmt.Sprintf("image@%s", digest)
+	imageOne  = fmt.Sprintf("image1@%s", digestOne)
+)
+
+var defaultArtifactCache = ArtifactCache{"hash": ImageDetails{
+	Digest: "digest",
+	ID:     "id",
+}}
+
+func mockHashForArtifact(hashes map[string]string) func(context.Context, build.Builder, *latest.Artifact) (string, error) {
+	return func(ctx context.Context, _ build.Builder, a *latest.Artifact) (string, error) {
+		return hashes[a.ImageName], nil
+	}
+}
+
+func Test_NewCache(t *testing.T) {
+	tests := []struct {
+		updateCacheFile   bool
+		needsPush         bool
+		updateClient      bool
+		name              string
+		opts              *config.SkaffoldOptions
+		expectedCache     *Cache
+		api               *testutil.FakeAPIClient
+		cacheFileContents interface{}
+	}{
+		{
+			name:              "get a valid cache from file",
+			cacheFileContents: defaultArtifactCache,
+			updateCacheFile:   true,
+			opts: &config.SkaffoldOptions{
+				CacheArtifacts: true,
+			},
+			updateClient: true,
+			api: &testutil.FakeAPIClient{
+				ImageSummaries: []types.ImageSummary{
+					{
+						ID: "image",
+					},
+				},
+			},
+			expectedCache: &Cache{
+				artifactCache: defaultArtifactCache,
+				useCache:      true,
+				imageList: []types.ImageSummary{
+					{
+						ID: "image",
+					},
+				},
+			},
+		},
+		{
+			name:              "needs push",
+			cacheFileContents: defaultArtifactCache,
+			needsPush:         true,
+			updateCacheFile:   true,
+			updateClient:      true,
+			opts: &config.SkaffoldOptions{
+				CacheArtifacts: true,
+			},
+			api: &testutil.FakeAPIClient{},
+			expectedCache: &Cache{
+				artifactCache: defaultArtifactCache,
+				useCache:      true,
+				needsPush:     true,
+			},
+		},
+		{
+			name:              "valid cache file exists, but useCache is false",
+			cacheFileContents: defaultArtifactCache,
+			api:               &testutil.FakeAPIClient{},
+			opts:              &config.SkaffoldOptions{},
+			expectedCache:     &Cache{},
+		},
+		{
+
+			name:              "corrupted cache file",
+			cacheFileContents: "corrupted cache file",
+			opts: &config.SkaffoldOptions{
+				CacheArtifacts: true,
+			},
+			expectedCache: &Cache{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			cacheFile := createTempCacheFile(t, test.cacheFileContents)
+			if test.updateCacheFile {
+				test.expectedCache.cacheFile = cacheFile
+			}
+			test.opts.CacheFile = cacheFile
+
+			originalDockerClient := newDockerCilent
+			newDockerCilent = func() (docker.LocalDaemon, error) {
+				return docker.NewLocalDaemon(test.api, nil), nil
+			}
+			defer func() {
+				newDockerCilent = originalDockerClient
+			}()
+
+			if test.updateClient {
+				test.expectedCache.client = docker.NewLocalDaemon(test.api, nil)
+			}
+
+			actualCache := NewCache(context.Background(), nil, test.opts, test.needsPush)
+
+			// cmp.Diff cannot access unexported fields, so use reflect.DeepEqual here directly
+			if !reflect.DeepEqual(test.expectedCache, actualCache) {
+				t.Errorf("Expected result different from actual result. Expected: %v, Actual: %v", test.expectedCache, actualCache)
+			}
+		})
+	}
+}
+
+func createTempCacheFile(t *testing.T, cacheFileContents interface{}) string {
+	temp, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("error creating temp cache file: %v", err)
+	}
+	defer temp.Close()
+	contents, err := yaml.Marshal(cacheFileContents)
+	if err != nil {
+		t.Fatalf("error marshalling cache: %v", err)
+	}
+	if err := ioutil.WriteFile(temp.Name(), contents, 0755); err != nil {
+		t.Fatalf("error writing contents to %s: %v", temp.Name(), err)
+	}
+	return temp.Name()
+}

pkg/skaffold/build/cache/hash.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"os"
+	"sort"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+	"github.com/pkg/errors"
+)
+
+var (
+	// For testing
+	hashFunction = cacheHasher
+)
+
+func getHashForArtifact(ctx context.Context, builder build.Builder, a *latest.Artifact) (string, error) {
+	deps, err := builder.DependenciesForArtifact(ctx, a)
+	if err != nil {
+		return "", errors.Wrapf(err, "getting dependencies for %s", a.ImageName)
+	}
+	sort.Strings(deps)
+	var hashes []string
+	for _, d := range deps {
+		h, err := hashFunction(d)
+		if err != nil {
+			return "", errors.Wrapf(err, "getting hash for %s", d)
+		}
+		hashes = append(hashes, h)
+	}
+	// get a key for the hashes
+	c := bytes.NewBuffer([]byte{})
+	enc := json.NewEncoder(c)
+	enc.Encode(hashes)
+	return util.SHA256(c)
+}
+
+// cacheHasher takes hashes the contents and name of a file
+func cacheHasher(p string) (string, error) {
+	h := md5.New()
+	fi, err := os.Lstat(p)
+	if err != nil {
+		return "", err
+	}
+	h.Write([]byte(fi.Mode().String()))
+	h.Write([]byte(fi.Name()))
+	if fi.Mode().IsRegular() {
+		f, err := os.Open(p)
+		if err != nil {
+			return "", err
+		}
+		defer f.Close()
+		if _, err := io.Copy(h, f); err != nil {
+			return "", err
+		}
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}