fix flake in in-cluster build

integration/build_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -123,24 +124,31 @@ func TestBuildInCluster(t *testing.T) {
 	}
 
 	testutil.Run(t, "", func(t *testutil.T) {
+		//this workaround is to ensure there is no overlap between testcases on kokoro
+		//see https://github.com/GoogleContainerTools/skaffold/issues/2781#issuecomment-527770537
+		tmpDir := t.NewTempDir()
+		testCaseDir := "testdata/skaffold-in-cluster"
+		workDir := path.Join(tmpDir.Root(), "testdata/skaffold-in-cluster")
+		t.CopyDir(testCaseDir, workDir)
+
 		// copy the skaffold binary to the test case folder
 		// this is geared towards the in-docker setup: the fresh built binary is here
 		// for manual testing, we can override this temporarily
 		skaffoldSrc, err := exec.LookPath("skaffold")
 		if err != nil {
 			t.Fatalf("failed to find skaffold binary: %s", err)
 		}
-		skaffoldDst := "./testdata/skaffold-in-cluster/skaffold"
+		skaffoldDst := path.Join(workDir, "skaffold")
 		t.CopyFile(skaffoldSrc, skaffoldDst)
 
 		ns, k8sClient, cleanupNs := SetupNamespace(t.T)
 		defer cleanupNs()
 
 		// TODO: until https://github.com/GoogleContainerTools/skaffold/issues/2757 is resolved, this is the simplest
 		// way to override the build.cluster.namespace
-		revert := replaceNamespace("./testdata/skaffold-in-cluster/skaffold.yaml", t, ns)
+		revert := replaceNamespace(path.Join(workDir, "skaffold.yaml"), t, ns)
 		defer revert()
-		revert = replaceNamespace("./testdata/skaffold-in-cluster/build-step/kustomization.yaml", t, ns)
+		revert = replaceNamespace(path.Join(workDir, "build-step/kustomization.yaml"), t, ns)
 		defer revert()
 
 		//we have to copy the e2esecret from default ns -> temporary namespace for kaniko
@@ -150,12 +158,12 @@ func TestBuildInCluster(t *testing.T) {
 		}
 		secret.Namespace = ns.Name
 		secret.ResourceVersion = ""
-		_, err = k8sClient.client.CoreV1().Secrets(ns.Name).Create(secret)
+		_, err = k8sClient.Secrets().Create(secret)
 		if err != nil {
 			t.Fatalf("failed creating %s/e2escret: %s", ns.Name, err)
 		}
 
-		logs := skaffold.Run("-p", "create-build-step", "--cache-artifacts=true").InDir("./testdata/skaffold-in-cluster").InNs(ns.Name).RunOrFailOutput(t.T)
+		logs := skaffold.Run("-p", "create-build-step", "--cache-artifacts=true").InDir(workDir).InNs(ns.Name).RunOrFailOutput(t.T)
 		t.Logf("create-build-step logs: \n%s", logs)
 
 		k8sClient.WaitForPodsInPhase(corev1.PodSucceeded, "skaffold-in-cluster")

integration/util.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	core_v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+
 	pkgkubernetes "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
@@ -80,6 +82,14 @@ type NSKubernetesClient struct {
 	ns     string
 }
 
+func (k *NSKubernetesClient) Pods() core_v1.PodInterface {
+	return k.client.CoreV1().Pods(k.ns)
+}
+
+func (k *NSKubernetesClient) Secrets() core_v1.SecretInterface {
+	return k.client.CoreV1().Secrets(k.ns)
+}
+
 // WaitForPodsReady waits for a list of pods to become ready.
 func (k *NSKubernetesClient) WaitForPodsReady(podNames ...string) {
 	k.WaitForPodsInPhase(v1.PodRunning, podNames...)

testutil/util.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path"
 	"reflect"
 	"regexp"
 	"strings"
@@ -66,12 +67,45 @@ func (t *T) CopyFile(src, dst string) {
 		t.Fatalf("failed to copy file %s to %s: %s", src, dst, err)
 	}
 	t.teardownActions = append(t.teardownActions, func() {
-		if err := os.Remove(dst); err != nil {
+		if err := os.Remove(dst); err != nil && !os.IsNotExist(err) {
 			t.Errorf("failed to remove %s: %s", dst, err)
 		}
 	})
 }
 
+func (t *T) CopyDir(src string, dst string) {
+	srcInfo, err := os.Stat(src)
+	if err != nil {
+		t.Fatalf("failed to copy dir %s->%s: %s ", src, dst, err)
+	}
+
+	if err = os.MkdirAll(dst, srcInfo.Mode()); err != nil {
+		t.Fatalf("failed to copy dir %s->%s: %s ", src, dst, err)
+	}
+
+	files, err := ioutil.ReadDir(src)
+	if err != nil {
+		t.Fatalf("failed to copy dir %s->%s: %s ", src, dst, err)
+	}
+
+	for _, f := range files {
+		srcfp := path.Join(src, f.Name())
+		dstfp := path.Join(dst, f.Name())
+
+		if f.IsDir() {
+			t.CopyDir(srcfp, dstfp)
+		} else {
+			t.CopyFile(srcfp, dstfp)
+		}
+	}
+	t.teardownActions = append(t.teardownActions, func() {
+		//by the time this callback is called, all the files should be removed
+		if err := os.Remove(dst); err != nil && !os.IsNotExist(err) {
+			t.Errorf("failed to remove dir %s: %s", dst, err)
+		}
+	})
+}
+
 func (t *T) CheckMatches(pattern, actual string) {
 	t.T.Helper()
 	if matches, _ := regexp.MatchString(pattern, actual); !matches {