feat: prototype control api devloop intent (#6636)

* feat: prototype control api devloop intent

* fix lint

* use separate callback

* reset devlop intent

* add an integration test

* fix lint

* fix integration test

* fix continuously

Author: tejal29

docs/content/en/api/skaffold.swagger.json

@@ -3535,6 +3535,10 @@
         "deploy": {
           "type": "boolean",
           "format": "boolean"
+        },
+        "devloop": {
+          "type": "boolean",
+          "format": "boolean"
         }
       },
       "description": "Intent represents user intents for a given phase."

docs/content/en/docs/references/api/grpc.md

@@ -384,6 +384,7 @@ Intent represents user intents for a given phase.
 | build | [bool](#bool) |  | in case skaffold dev is ran with autoBuild=false, a build intent enables building once |
 | sync | [bool](#bool) |  | in case skaffold dev is ran with autoSync=false, a sync intent enables file sync once |
 | deploy | [bool](#bool) |  | in case skaffold dev is ran with autoDeploy=false, a deploy intent enables deploys once |
+| devloop | [bool](#bool) |  | in case skaffold dev is ran with autoDeploy=false, autoSync=false and autoBuild=false a devloop intent enables the entire dev loop once |
 
 
 

integration/control_api_test.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
+	"github.com/GoogleContainerTools/skaffold/proto/v1"
+)
+
+const (
+	testDev = "test-dev"
+)
+
+func TestControlAPIManualTriggers(t *testing.T) {
+	MarkIntegrationTest(t, CanRunWithoutGcp)
+
+	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
+	defer Run(t, "testdata/dev", "rm", "foo")
+
+	ns, client := SetupNamespace(t)
+	out := bytes.Buffer{}
+	rpcAddr := randomPort()
+	skaffold.Dev("--auto-build=false", "--auto-sync=false", "--auto-deploy=false", "--rpc-port", rpcAddr, "--cache-artifacts=false").InDir("testdata/dev").InNs(ns.Name).RunInBackgroundWithOutput(t, &out)
+
+	rpcClient, entries := apiEvents(t, rpcAddr)
+
+	// throw away first 5 entries of log (from first run of dev loop)
+	for i := 0; i < 5; i++ {
+		<-entries
+	}
+
+	dep := client.GetDeployment(testDev)
+
+	// Make a change to foo
+	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
+
+	// Execute dev loop trigger
+	rpcClient.Execute(context.Background(), &proto.UserIntentRequest{
+		Intent: &proto.Intent{
+			Devloop: true,
+		},
+	})
+	// Ensure we see a build triggered in the event log
+	err := wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
+		e := <-entries
+		return e.GetEvent().GetBuildEvent().GetArtifact() == testDev, nil
+	})
+	failNowIfError(t, err)
+	// verify deployment happened.
+	verifyDeployment(t, entries, client, dep)
+
+	// Make another change to foo and we should not see any event log.
+	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
+
+	// Give skaffold some time to register a file change.
+	time.Sleep(1 * time.Second)
+	if c := strings.Count(out.String(), "Generating tags"); c != 2 {
+		failNowIfError(t, fmt.Errorf("expected to see tags generated twice (1st build and 1 trigger), saw %d times", c))
+	}
+}

integration/dev_test.go

@@ -68,14 +68,14 @@ func TestDevNotification(t *testing.T) {
 
 			skaffold.Dev("--trigger", test.trigger).InDir("testdata/dev").InNs(ns.Name).RunBackground(t)
 
-			dep := client.GetDeployment("test-dev")
+			dep := client.GetDeployment(testDev)
 
 			// Make a change to foo so that dev is forced to delete the Deployment and redeploy
 			Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
 
 			// Make sure the old Deployment and the new Deployment are different
 			err := wait.PollImmediate(time.Millisecond*500, 1*time.Minute, func() (bool, error) {
-				newDep := client.GetDeployment("test-dev")
+				newDep := client.GetDeployment(testDev)
 				logrus.Infof("old gen: %d, new gen: %d", dep.GetGeneration(), newDep.GetGeneration())
 				return dep.GetGeneration() != newDep.GetGeneration(), nil
 			})
@@ -157,7 +157,7 @@ func TestDevAPITriggers(t *testing.T) {
 		<-entries
 	}
 
-	dep := client.GetDeployment("test-dev")
+	dep := client.GetDeployment(testDev)
 
 	// Make a change to foo
 	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
@@ -172,7 +172,7 @@ func TestDevAPITriggers(t *testing.T) {
 	// Ensure we see a build triggered in the event log
 	err := wait.PollImmediate(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
 		e := <-entries
-		return e.GetEvent().GetBuildEvent().GetArtifact() == "test-dev", nil
+		return e.GetEvent().GetBuildEvent().GetArtifact() == testDev, nil
 	})
 	failNowIfError(t, err)
 
@@ -207,7 +207,7 @@ func TestDevAPIAutoTriggers(t *testing.T) {
 		<-entries
 	}
 
-	dep := client.GetDeployment("test-dev")
+	dep := client.GetDeployment(testDev)
 
 	// Make a change to foo
 	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
@@ -223,7 +223,7 @@ func TestDevAPIAutoTriggers(t *testing.T) {
 	// Ensure we see a build triggered in the event log
 	err := wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
 		e := <-entries
-		return e.GetEvent().GetBuildEvent().GetArtifact() == "test-dev", nil
+		return e.GetEvent().GetBuildEvent().GetArtifact() == testDev, nil
 	})
 	failNowIfError(t, err)
 
@@ -247,7 +247,7 @@ func verifyDeployment(t *testing.T, entries chan *proto.LogEntry, client *NSKube
 
 	// Make sure the old Deployment and the new Deployment are different
 	err = wait.Poll(time.Millisecond*500, 1*time.Minute, func() (bool, error) {
-		newDep := client.GetDeployment("test-dev")
+		newDep := client.GetDeployment(testDev)
 		logrus.Infof("old gen: %d, new gen: %d", dep.GetGeneration(), newDep.GetGeneration())
 		return dep.GetGeneration() != newDep.GetGeneration(), nil
 	})

integration/skaffold/helper.go

@@ -137,7 +137,11 @@ func GeneratePipeline(args ...string) *RunBuilder {
 }
 
 func withDefaults(command string, args []string) *RunBuilder {
-	return &RunBuilder{command: command, args: args, repo: "gcr.io/k8s-skaffold"}
+	repo := os.Getenv("DEFAULT_REPO")
+	if repo == "" {
+		repo = "gcr.io/k8s-skaffold"
+	}
+	return &RunBuilder{command: command, args: args, repo: repo}
 }
 
 // InDir sets the directory in which skaffold is running.
@@ -197,6 +201,9 @@ func (b *RunBuilder) RunBackground(t *testing.T) {
 }
 
 // RunLive runs the skaffold command in the background with live output.
+// !!Warning!! RunLive blocks the skaffold command until the caller reads from
+// the returned `PipeReader`. Please use `WaitForLogs` or similar to read
+// continuously from the returned `PipeReader`.
 func (b *RunBuilder) RunLive(t *testing.T) io.ReadCloser {
 	t.Helper()
 	pr, pw := io.Pipe()
@@ -207,6 +214,12 @@ func (b *RunBuilder) RunLive(t *testing.T) io.ReadCloser {
 	return pr
 }
 
+// RunInBackgroundWithOutput runs the skaffold command in the background.
+func (b *RunBuilder) RunInBackgroundWithOutput(t *testing.T, out io.Writer) {
+	t.Helper()
+	b.runForked(t, out)
+}
+
 // runForked runs the skaffold command in the background with stdout sent to the provided writer.
 func (b *RunBuilder) runForked(t *testing.T, out io.Writer) {
 	t.Helper()

pkg/skaffold/runner/intent.go

@@ -16,12 +16,15 @@ limitations under the License.
 
 package runner
 
-import "sync"
+import (
+	"sync"
+)
 
 type Intents struct {
 	build      bool
 	sync       bool
 	deploy     bool
+	devloop    bool
 	autoBuild  bool
 	autoSync   bool
 	autoDeploy bool
@@ -88,6 +91,12 @@ func (i *Intents) SetDeploy(val bool) {
 	i.lock.Unlock()
 }
 
+func (i *Intents) SetDevloop(val bool) {
+	i.lock.Lock()
+	i.devloop = val
+	i.lock.Unlock()
+}
+
 func (i *Intents) GetAutoBuild() bool {
 	i.lock.Lock()
 	defer i.lock.Unlock()
@@ -106,6 +115,12 @@ func (i *Intents) GetAutoDeploy() bool {
 	return i.autoDeploy
 }
 
+func (i *Intents) GetAutoDevloop() bool {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+	return i.autoDeploy && i.autoBuild && i.autoSync
+}
+
 func (i *Intents) SetAutoBuild(val bool) {
 	i.lock.Lock()
 	i.autoBuild = val
@@ -124,10 +139,24 @@ func (i *Intents) SetAutoDeploy(val bool) {
 	i.lock.Unlock()
 }
 
-// returns build, sync, and deploy intents (in that order)
+func (i *Intents) SetAutoDevloop(val bool) {
+	i.lock.Lock()
+	i.autoDeploy = val
+	i.autoSync = val
+	i.autoBuild = val
+	i.lock.Unlock()
+}
+
+// GetIntents returns build, sync, and deploy intents (in that order)
+// If intent is devloop intent, all are returned true
 func (i *Intents) GetIntents() (bool, bool, bool) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
+	if i.devloop {
+		// reset Devloop intent
+		i.devloop = false
+		return true, true, true
+	}
 	return i.build, i.sync, i.deploy
 }
 

pkg/skaffold/runner/v1/dev.go

@@ -55,13 +55,14 @@ func (r *SkaffoldRunner) doDev(ctx context.Context, out io.Writer) error {
 
 	buildIntent, syncIntent, deployIntent := r.intents.GetIntents()
 	log.Entry(ctx).Tracef("dev intents: build %t, sync %t, deploy %t\n", buildIntent, syncIntent, deployIntent)
-	needsSync := syncIntent && len(r.changeSet.NeedsResync()) > 0
 	needsBuild := buildIntent && len(r.changeSet.NeedsRebuild()) > 0
+	needsSync := syncIntent && (len(r.changeSet.NeedsResync()) > 0 || needsBuild)
 	needsTest := len(r.changeSet.NeedsRetest()) > 0
-	needsDeploy := deployIntent && r.changeSet.NeedsRedeploy()
+	needsDeploy := deployIntent && (r.changeSet.NeedsRedeploy() || needsBuild)
 	if !needsSync && !needsBuild && !needsTest && !needsDeploy {
 		return nil
 	}
+	log.Entry(ctx).Debugf(" devloop: build %t, sync %t, deploy %t\n", needsBuild, needsSync, needsDeploy)
 
 	r.deployer.GetLogger().Mute()
 	// if any action is going to be performed, reset the monitor's changed component tracker for debouncing

pkg/skaffold/runner/v1/new.go

@@ -150,6 +150,8 @@ func setupIntents(runCtx *runcontext.RunContext) (*runner.Intents, chan bool) {
 	setupTrigger("build", intents.SetBuild, intents.SetAutoBuild, intents.GetAutoBuild, server.SetBuildCallback, server.SetAutoBuildCallback, intentChan)
 	setupTrigger("sync", intents.SetSync, intents.SetAutoSync, intents.GetAutoSync, server.SetSyncCallback, server.SetAutoSyncCallback, intentChan)
 	setupTrigger("deploy", intents.SetDeploy, intents.SetAutoDeploy, intents.GetAutoDeploy, server.SetDeployCallback, server.SetAutoDeployCallback, intentChan)
+	// Setup callback function to buildCallback since build is the start of the devloop.
+	setupTrigger("devloop", intents.SetDevloop, intents.SetAutoDevloop, intents.GetAutoDevloop, server.SetDevloopCallback, server.SetAutoDevloopCallback, intentChan)
 
 	return intents, intentChan
 }

pkg/skaffold/server/endpoints.go

@@ -45,6 +45,14 @@ func (s *server) Handle(ctx context.Context, e *proto.Event) (*empty.Empty, erro
 }
 
 func (s *server) Execute(ctx context.Context, intent *proto.UserIntentRequest) (*empty.Empty, error) {
+	if intent.GetIntent().GetDevloop() {
+		event.ResetStateOnBuild()
+		go func() {
+			s.devloopIntentCallback()
+		}()
+		return &empty.Empty{}, nil
+	}
+
 	if intent.GetIntent().GetBuild() {
 		event.ResetStateOnBuild()
 		go func() {

pkg/skaffold/server/server.go

@@ -47,12 +47,14 @@ var (
 )
 
 type server struct {
-	buildIntentCallback  func()
-	syncIntentCallback   func()
-	deployIntentCallback func()
-	autoBuildCallback    func(bool)
-	autoSyncCallback     func(bool)
-	autoDeployCallback   func(bool)
+	buildIntentCallback   func()
+	syncIntentCallback    func()
+	deployIntentCallback  func()
+	devloopIntentCallback func()
+	autoBuildCallback     func(bool)
+	autoSyncCallback      func(bool)
+	autoDeployCallback    func(bool)
+	autoDevloopCallback   func(bool)
 }
 
 func SetBuildCallback(callback func()) {
@@ -61,6 +63,12 @@ func SetBuildCallback(callback func()) {
 	}
 }
 
+func SetDevloopCallback(callback func()) {
+	if srv != nil {
+		srv.devloopIntentCallback = callback
+	}
+}
+
 func SetDeployCallback(callback func()) {
 	if srv != nil {
 		srv.deployIntentCallback = callback
@@ -79,6 +87,12 @@ func SetAutoBuildCallback(callback func(bool)) {
 	}
 }
 
+func SetAutoDevloopCallback(callback func(bool)) {
+	if srv != nil {
+		srv.autoDevloopCallback = callback
+	}
+}
+
 func SetAutoDeployCallback(callback func(bool)) {
 	if srv != nil {
 		srv.autoDeployCallback = callback
@@ -164,20 +178,24 @@ func newGRPCServer(preferredPort int) (func() error, int, error) {
 
 	s := grpc.NewServer()
 	srv = &server{
-		buildIntentCallback:  func() {},
-		deployIntentCallback: func() {},
-		syncIntentCallback:   func() {},
-		autoBuildCallback:    func(bool) {},
-		autoSyncCallback:     func(bool) {},
-		autoDeployCallback:   func(bool) {},
+		buildIntentCallback:   func() {},
+		deployIntentCallback:  func() {},
+		syncIntentCallback:    func() {},
+		devloopIntentCallback: func() {},
+		autoBuildCallback:     func(bool) {},
+		autoSyncCallback:      func(bool) {},
+		autoDeployCallback:    func(bool) {},
+		autoDevloopCallback:   func(bool) {},
 	}
 	v2.Srv = &v2.Server{
-		BuildIntentCallback:  func() {},
-		DeployIntentCallback: func() {},
-		SyncIntentCallback:   func() {},
-		AutoBuildCallback:    func(bool) {},
-		AutoSyncCallback:     func(bool) {},
-		AutoDeployCallback:   func(bool) {},
+		BuildIntentCallback:   func() {},
+		DeployIntentCallback:  func() {},
+		SyncIntentCallback:    func() {},
+		DevloopIntentCallback: func() {},
+		AutoBuildCallback:     func(bool) {},
+		AutoSyncCallback:      func(bool) {},
+		AutoDeployCallback:    func(bool) {},
+		AutoDevloopCallback:   func(bool) {},
 	}
 	protoV1.RegisterSkaffoldServiceServer(s, srv)
 	protoV2.RegisterSkaffoldV2ServiceServer(s, v2.Srv)