Merge pull request #1013 from nkubala/test-runner

Add test runner

Author: dgageot

deploy/skaffold/Dockerfile

@@ -47,6 +47,11 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8
 RUN apt-get update \
   && apt-get install -y bazel
 
+ENV CONTAINER_STRUCTURE_TEST_VERSION=1.5.0
+RUN curl -LO https://storage.googleapis.com/container-structure-test/v${CONTAINER_STRUCTURE_TEST_VERSION}/container-structure-test-linux-amd64 \
+  && chmod +x container-structure-test-linux-amd64 \
+  && mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
+
 ENV PATH /usr/local/go/bin:/go/bin:/google-cloud-sdk/bin:$PATH
 
 FROM runtime_deps as builder
@@ -94,4 +99,3 @@ CMD ["make", "integration"]
 FROM runtime_deps as distribution
 
 COPY --from=integration /usr/bin/skaffold /usr/bin/skaffold
-

integration/examples/getting-started/skaffold.yaml

@@ -3,6 +3,10 @@ kind: Config
 build:
   artifacts:
   - image: gcr.io/k8s-skaffold/skaffold-example
+test:
+  - image: gcr.io/k8s-skaffold/skaffold-example
+    structureTests:
+      - ./test/*
 deploy:
   kubectl:
     manifests:
@@ -12,3 +16,8 @@ profiles:
     build:
       googleCloudBuild:
         projectId: k8s-skaffold
+  - name: test
+    test:
+      - image: gcr.io/k8s-skaffold/skaffold-example
+        structureTests:
+          - ./test/profile_structure_test.yaml

integration/examples/getting-started/test/profile_structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no go binary'
+    path: '/usr/bin/go'
+    shouldExist: false

integration/examples/getting-started/test/structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no local go binary'
+    path: /usr/local/bin/go'
+    shouldExist: false

pkg/skaffold/runner/runner.go

@@ -38,6 +38,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
 
 	"github.com/pkg/errors"
@@ -51,6 +52,7 @@ var ErrorConfigurationChanged = errors.New("configuration changed")
 type SkaffoldRunner struct {
 	build.Builder
 	deploy.Deployer
+	test.Tester
 	tag.Tagger
 
 	opts         *config.SkaffoldOptions
@@ -76,19 +78,25 @@ func NewForConfig(opts *config.SkaffoldOptions, cfg *latest.SkaffoldConfig) (*Sk
 		return nil, errors.Wrap(err, "parsing skaffold build config")
 	}
 
+	tester, err := getTester(&cfg.Test)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing skaffold test config")
+	}
+
 	deployer, err := getDeployer(&cfg.Deploy, kubeContext, opts.Namespace)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing skaffold deploy config")
 	}
 
 	deployer = deploy.WithLabels(deployer, opts, builder, deployer, tagger)
-	builder, deployer = WithTimings(builder, deployer)
+	builder, tester, deployer = WithTimings(builder, tester, deployer)
 	if opts.Notification {
 		deployer = WithNotification(deployer)
 	}
 
 	return &SkaffoldRunner{
 		Builder:      builder,
+		Tester:       tester,
 		Deployer:     deployer,
 		Tagger:       tagger,
 		opts:         opts,
@@ -115,6 +123,10 @@ func getBuilder(cfg *latest.BuildConfig, kubeContext string) (build.Builder, err
 	}
 }
 
+func getTester(cfg *[]latest.TestCase) (test.Tester, error) {
+	return test.NewTester(cfg)
+}
+
 func getDeployer(cfg *latest.DeployConfig, kubeContext string, namespace string) (deploy.Deployer, error) {
 	deployers := []deploy.Deployer{}
 
@@ -171,13 +183,17 @@ func getTagger(t latest.TagPolicy, customTag string) (tag.Tagger, error) {
 	}
 }
 
-// Run builds artifacts and then deploys them.
+// Run builds artifacts, runs tests on built artifacts, and then deploys them.
 func (r *SkaffoldRunner) Run(ctx context.Context, out io.Writer, artifacts []*latest.Artifact) error {
 	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
 	if err != nil {
 		return errors.Wrap(err, "build step")
 	}
 
+	if err = r.Test(out, bRes); err != nil {
+		return errors.Wrap(err, "test step")
+	}
+
 	_, err = r.Deploy(ctx, out, bRes)
 	if err != nil {
 		return errors.Wrap(err, "deploy step")
@@ -241,12 +257,20 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 			}
 
 			r.updateBuiltImages(imageList, bRes)
+			if err := r.Test(out, bRes); err != nil {
+				logrus.Warnln("Skipping Deploy due to failed tests:", err)
+				return nil
+			}
 
 			if _, err = r.Deploy(ctx, out, r.builds); err != nil {
 				logrus.Warnln("Skipping Deploy due to error:", err)
 				return nil
 			}
 		case changed.needsRedeploy:
+			if err := r.Test(out, r.builds); err != nil {
+				logrus.Warnln("Skipping Deploy due to failed tests:", err)
+				return nil
+			}
 			if _, err := r.Deploy(ctx, out, r.builds); err != nil {
 				logrus.Warnln("Skipping Deploy due to error:", err)
 				return nil
@@ -275,6 +299,14 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 		}
 	}
 
+	// Watch test configuration
+	if err := watcher.Register(
+		func() ([]string, error) { return r.TestDependencies(), nil },
+		func(watch.Events) { changed.needsRedeploy = true },
+	); err != nil {
+		return nil, errors.Wrap(err, "watching test files")
+	}
+
 	// Watch deployment configuration
 	if err := watcher.Register(
 		func() ([]string, error) { return r.Dependencies() },
@@ -298,6 +330,9 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 	}
 
 	r.updateBuiltImages(imageList, bRes)
+	if err := r.Test(out, bRes); err != nil {
+		return nil, errors.Wrap(err, "exiting dev mode because the first test run failed")
+	}
 
 	_, err = r.Deploy(ctx, out, r.builds)
 	if err != nil {

pkg/skaffold/runner/runner_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
 	"github.com/GoogleContainerTools/skaffold/testutil"
 	clientgo "k8s.io/client-go/kubernetes"
@@ -65,6 +66,23 @@ func (t *TestBuilder) Build(ctx context.Context, w io.Writer, tagger tag.Tagger,
 	return builds, nil
 }
 
+type TestTester struct {
+	errors []error
+}
+
+func (t *TestTester) Test(out io.Writer, builds []build.Artifact) error {
+	if len(t.errors) > 0 {
+		err := t.errors[0]
+		t.errors = t.errors[1:]
+		return err
+	}
+	return nil
+}
+
+func (t *TestTester) TestDependencies() []string {
+	return []string{}
+}
+
 type TestDeployer struct {
 	deployed []build.Artifact
 	errors   []error
@@ -140,6 +158,7 @@ func TestNewForConfig(t *testing.T) {
 		config           *latest.SkaffoldConfig
 		shouldErr        bool
 		expectedBuilder  build.Builder
+		expectedTester   test.Tester
 		expectedDeployer deploy.Deployer
 	}{
 		{
@@ -158,6 +177,7 @@ func TestNewForConfig(t *testing.T) {
 				},
 			},
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -184,6 +204,7 @@ func TestNewForConfig(t *testing.T) {
 			},
 			shouldErr:        true,
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -197,6 +218,7 @@ func TestNewForConfig(t *testing.T) {
 				}},
 			shouldErr:        true,
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -218,9 +240,10 @@ func TestNewForConfig(t *testing.T) {
 
 			testutil.CheckError(t, test.shouldErr, err)
 			if cfg != nil {
-				b, d := WithTimings(test.expectedBuilder, test.expectedDeployer)
+				b, _t, d := WithTimings(test.expectedBuilder, test.expectedTester, test.expectedDeployer)
 
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, b, cfg.Builder)
+				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, _t, cfg.Tester)
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, d, cfg.Deployer)
 			}
 		})
@@ -232,13 +255,15 @@ func TestRun(t *testing.T) {
 		description string
 		config      *latest.SkaffoldConfig
 		builder     build.Builder
+		tester      test.Tester
 		deployer    deploy.Deployer
 		shouldErr   bool
 	}{
 		{
 			description: "run no error",
 			config:      &latest.SkaffoldConfig{},
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer:    &TestDeployer{},
 		},
 		{
@@ -247,6 +272,7 @@ func TestRun(t *testing.T) {
 			builder: &TestBuilder{
 				errors: []error{fmt.Errorf("")},
 			},
+			tester:    &TestTester{},
 			shouldErr: true,
 		},
 		{
@@ -261,17 +287,42 @@ func TestRun(t *testing.T) {
 				},
 			},
 			builder: &TestBuilder{},
+			tester:  &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{fmt.Errorf("")},
 			},
 			shouldErr: true,
 		},
+		{
+			description: "run test error",
+			config: &latest.SkaffoldConfig{
+				Build: latest.BuildConfig{
+					Artifacts: []*latest.Artifact{
+						{
+							ImageName: "test",
+						},
+					},
+				},
+				Test: []latest.TestCase{
+					{
+						ImageName:      "test",
+						StructureTests: []string{"fake_file.yaml"},
+					},
+				},
+			},
+			builder: &TestBuilder{},
+			tester: &TestTester{
+				errors: []error{fmt.Errorf("")},
+			},
+			shouldErr: true,
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
 			runner := &SkaffoldRunner{
 				Builder:  test.builder,
+				Tester:   test.tester,
 				Deployer: test.deployer,
 				Tagger:   &tag.ChecksumTagger{},
 				opts:     &config.SkaffoldOptions{},
@@ -290,6 +341,7 @@ func TestDev(t *testing.T) {
 	var tests = []struct {
 		description    string
 		builder        build.Builder
+		tester         test.Tester
 		deployer       deploy.Deployer
 		watcherFactory watch.Factory
 		shouldErr      bool
@@ -306,23 +358,35 @@ func TestDev(t *testing.T) {
 		{
 			description: "fails to deploy the first time",
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{fmt.Errorf("")},
 			},
 			watcherFactory: NewWatcherFactory(nil, nil),
 			shouldErr:      true,
 		},
+		{
+			description: "fails to deploy due to failed tests",
+			builder:     &TestBuilder{},
+			tester: &TestTester{
+				errors: []error{fmt.Errorf("")},
+			},
+			watcherFactory: NewWatcherFactory(nil, nil),
+			shouldErr:      true,
+		},
 		{
 			description: "ignore subsequent build errors",
 			builder: &TestBuilder{
 				errors: []error{nil, fmt.Errorf("")},
 			},
+			tester:         &TestTester{},
 			deployer:       &TestDeployer{},
 			watcherFactory: NewWatcherFactory(nil, nil, nil),
 		},
 		{
 			description: "ignore subsequent deploy errors",
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{nil, fmt.Errorf("")},
 			},
@@ -331,6 +395,7 @@ func TestDev(t *testing.T) {
 		{
 			description:    "fail to watch files",
 			builder:        &TestBuilder{},
+			tester:         &TestTester{},
 			deployer:       &TestDeployer{},
 			watcherFactory: NewWatcherFactory(fmt.Errorf(""), nil),
 			shouldErr:      true,
@@ -341,6 +406,7 @@ func TestDev(t *testing.T) {
 		t.Run(test.description, func(t *testing.T) {
 			runner := &SkaffoldRunner{
 				Builder:      test.builder,
+				Tester:       test.tester,
 				Deployer:     test.deployer,
 				Tagger:       &tag.ChecksumTagger{},
 				watchFactory: test.watcherFactory,
@@ -360,6 +426,7 @@ func TestBuildAndDeployAllArtifacts(t *testing.T) {
 	defer resetClient()
 
 	builder := &TestBuilder{}
+	tester := &TestTester{}
 	deployer := &TestDeployer{}
 	artifacts := []*latest.Artifact{
 		{ImageName: "image1"},
@@ -368,6 +435,7 @@ func TestBuildAndDeployAllArtifacts(t *testing.T) {
 
 	runner := &SkaffoldRunner{
 		Builder:  builder,
+		Tester:   tester,
 		Deployer: deployer,
 		opts:     &config.SkaffoldOptions{},
 	}