feat: add --tolerate-failures-until-deadline flag and deploy.tolerateFailuresUntilDeadline config for improved ci/cd usage (#8047)

Author: aaron-prindle

cmd/skaffold/app/cmd/flags.go

@@ -332,6 +332,15 @@ var flagRegistry = []Flag{
 		DefinedOn:     []string{"dev", "debug", "deploy", "run", "apply"},
 		IsEnum:        true,
 	},
+	{
+		Name:          "tolerate-failures-until-deadline",
+		Usage:         "Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'",
+		Value:         &opts.TolerateFailuresStatusCheck,
+		DefValue:      false,
+		FlagAddMethod: "BoolVar",
+		DefinedOn:     []string{"dev", "debug", "deploy", "run", "apply"},
+		IsEnum:        true,
+	},
 	{
 		Name:          "fast-fail-status-check",
 		Usage:         "Configures `status-check` to fail immediately if any error occurs.  Otherwise `status-check` will attempt to check all resources once and only then report errors and possibly exit.  Defaults to 'true'",

docs-v2/content/en/schemas/v4beta1.json

@@ -1422,6 +1422,12 @@
           "type": "integer",
           "description": "*beta* deadline for deployments to stabilize in seconds.",
           "x-intellij-html-description": "<em>beta</em> deadline for deployments to stabilize in seconds."
+        },
+        "tolerateFailuresUntilDeadline": {
+          "type": "boolean",
+          "description": "configures the Skaffold \"status-check\" to tolerate failures (flapping deployments, etc.) until the statusCheckDeadlineSeconds duration or k8s object timeouts such as progressDeadlineSeconds, etc.",
+          "x-intellij-html-description": "configures the Skaffold &quot;status-check&quot; to tolerate failures (flapping deployments, etc.) until the statusCheckDeadlineSeconds duration or k8s object timeouts such as progressDeadlineSeconds, etc.",
+          "default": "false"
         }
       },
       "preferredOrder": [
@@ -1432,6 +1438,7 @@
         "cloudrun",
         "statusCheck",
         "statusCheckDeadlineSeconds",
+        "tolerateFailuresUntilDeadline",
         "kubeContext",
         "logs"
       ],

docs/content/en/docs/references/cli/_index.md

@@ -139,6 +139,7 @@ Options:
       --status-check=: Wait for deployed resources to stabilize
       --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories.
       --tail=false: Stream logs from deployed objects
+      --tolerate-failures-until-deadline=false: Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'
       --wait-for-connection=false: Blocks ending execution of skaffold until the /v2/events gRPC/HTTP endpoint is hit
 
 Usage:
@@ -168,6 +169,7 @@ Env vars:
 * `SKAFFOLD_STATUS_CHECK` (same as `--status-check`)
 * `SKAFFOLD_SYNC_REMOTE_CACHE` (same as `--sync-remote-cache`)
 * `SKAFFOLD_TAIL` (same as `--tail`)
+* `SKAFFOLD_TOLERATE_FAILURES_UNTIL_DEADLINE` (same as `--tolerate-failures-until-deadline`)
 * `SKAFFOLD_WAIT_FOR_CONNECTION` (same as `--wait-for-connection`)
 
 ### skaffold build
@@ -453,6 +455,7 @@ Options:
       --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories.
   -t, --tag='': The optional custom tag to use for images which overrides the current Tagger configuration
       --tail=true: Stream logs from deployed objects
+      --tolerate-failures-until-deadline=false: Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'
       --toot=false: Emit a terminal beep after the deploy is complete
       --trigger='notify': How is change detection triggered? (polling, notify, or manual)
       --wait-for-connection=false: Blocks ending execution of skaffold until the /v2/events gRPC/HTTP endpoint is hit
@@ -516,6 +519,7 @@ Env vars:
 * `SKAFFOLD_SYNC_REMOTE_CACHE` (same as `--sync-remote-cache`)
 * `SKAFFOLD_TAG` (same as `--tag`)
 * `SKAFFOLD_TAIL` (same as `--tail`)
+* `SKAFFOLD_TOLERATE_FAILURES_UNTIL_DEADLINE` (same as `--tolerate-failures-until-deadline`)
 * `SKAFFOLD_TOOT` (same as `--toot`)
 * `SKAFFOLD_TRIGGER` (same as `--trigger`)
 * `SKAFFOLD_WAIT_FOR_CONNECTION` (same as `--wait-for-connection`)
@@ -633,6 +637,7 @@ Options:
       --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories.
   -t, --tag='': The optional custom tag to use for images which overrides the current Tagger configuration
       --tail=false: Stream logs from deployed objects
+      --tolerate-failures-until-deadline=false: Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'
       --toot=false: Emit a terminal beep after the deploy is complete
       --wait-for-connection=false: Blocks ending execution of skaffold until the /v2/events gRPC/HTTP endpoint is hit
       --wait-for-deletions=true: Wait for pending deletions to complete before a deployment
@@ -681,6 +686,7 @@ Env vars:
 * `SKAFFOLD_SYNC_REMOTE_CACHE` (same as `--sync-remote-cache`)
 * `SKAFFOLD_TAG` (same as `--tag`)
 * `SKAFFOLD_TAIL` (same as `--tail`)
+* `SKAFFOLD_TOLERATE_FAILURES_UNTIL_DEADLINE` (same as `--tolerate-failures-until-deadline`)
 * `SKAFFOLD_TOOT` (same as `--toot`)
 * `SKAFFOLD_WAIT_FOR_CONNECTION` (same as `--wait-for-connection`)
 * `SKAFFOLD_WAIT_FOR_DELETIONS` (same as `--wait-for-deletions`)
@@ -740,6 +746,7 @@ Options:
       --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories.
   -t, --tag='': The optional custom tag to use for images which overrides the current Tagger configuration
       --tail=true: Stream logs from deployed objects
+      --tolerate-failures-until-deadline=false: Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'
       --toot=false: Emit a terminal beep after the deploy is complete
       --trigger='notify': How is change detection triggered? (polling, notify, or manual)
       --wait-for-connection=false: Blocks ending execution of skaffold until the /v2/events gRPC/HTTP endpoint is hit
@@ -803,6 +810,7 @@ Env vars:
 * `SKAFFOLD_SYNC_REMOTE_CACHE` (same as `--sync-remote-cache`)
 * `SKAFFOLD_TAG` (same as `--tag`)
 * `SKAFFOLD_TAIL` (same as `--tail`)
+* `SKAFFOLD_TOLERATE_FAILURES_UNTIL_DEADLINE` (same as `--tolerate-failures-until-deadline`)
 * `SKAFFOLD_TOOT` (same as `--toot`)
 * `SKAFFOLD_TRIGGER` (same as `--trigger`)
 * `SKAFFOLD_WAIT_FOR_CONNECTION` (same as `--wait-for-connection`)
@@ -1095,6 +1103,7 @@ Options:
       --sync-remote-cache='always': Controls how Skaffold manages the remote config cache (see `remote-cache-dir`). One of `always` (default), `missing`, or `never`. `always` syncs remote repositories to latest on access. `missing` only clones remote repositories if they do not exist locally. `never` means the user takes responsibility for updating remote repositories.
   -t, --tag='': The optional custom tag to use for images which overrides the current Tagger configuration
       --tail=false: Stream logs from deployed objects
+      --tolerate-failures-until-deadline=false: Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds  Otherwise deployment failures skaffold encounters will immediately fail the deployment.  Defaults to 'false'
       --toot=false: Emit a terminal beep after the deploy is complete
       --wait-for-connection=false: Blocks ending execution of skaffold until the /v2/events gRPC/HTTP endpoint is hit
       --wait-for-deletions=true: Wait for pending deletions to complete before a deployment
@@ -1153,6 +1162,7 @@ Env vars:
 * `SKAFFOLD_SYNC_REMOTE_CACHE` (same as `--sync-remote-cache`)
 * `SKAFFOLD_TAG` (same as `--tag`)
 * `SKAFFOLD_TAIL` (same as `--tail`)
+* `SKAFFOLD_TOLERATE_FAILURES_UNTIL_DEADLINE` (same as `--tolerate-failures-until-deadline`)
 * `SKAFFOLD_TOOT` (same as `--toot`)
 * `SKAFFOLD_WAIT_FOR_CONNECTION` (same as `--wait-for-connection`)
 * `SKAFFOLD_WAIT_FOR_DELETIONS` (same as `--wait-for-deletions`)

integration/run_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package integration
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -247,6 +248,44 @@ func TestRunTailDefaultNamespace(t *testing.T) {
 	}
 }
 
+func TestRunTailTolerateFailuresUntilDeadline(t *testing.T) {
+	MarkIntegrationTest(t, CanRunWithoutGcp)
+	var tsts = []struct {
+		description  string
+		dir          string
+		args         []string
+		deployments  []string
+		env          []string
+		targetLogOne string
+		targetLogTwo string
+	}{
+		{
+			description:  "status-check-tolerance",
+			dir:          "testdata/status-check-tolerance",
+			args:         []string{"--tolerate-failures-until-deadline"},
+			deployments:  []string{"tolerance-check"},
+			targetLogOne: "container will exit with error",
+			targetLogTwo: "Hello world!",
+			env:          []string{fmt.Sprintf("STOP_FAILING_TIME=%d", time.Now().Unix()+10)},
+		},
+	}
+
+	for _, test := range tsts {
+		t.Run(test.description, func(t *testing.T) {
+			if test.targetLogOne == "" || test.targetLogTwo == "" {
+				t.SkipNow()
+			}
+			ns, _ := SetupNamespace(t)
+
+			args := append(test.args, "--tail")
+			out := skaffold.Run(args...).InDir(test.dir).InNs(ns.Name).WithEnv(test.env).RunLive(t)
+			defer skaffold.Delete().InDir(test.dir).WithEnv(test.env).RunOrFail(t)
+			WaitForLogs(t, out, test.targetLogOne)
+			WaitForLogs(t, out, test.targetLogTwo)
+		})
+	}
+}
+
 func TestRunRenderOnly(t *testing.T) {
 	MarkIntegrationTest(t, CanRunWithoutGcp)
 

integration/testdata/status-check-tolerance/skaffold.yaml

@@ -0,0 +1,14 @@
+apiVersion: skaffold/v2beta29
+kind: Config
+build:
+  artifacts:
+    - image: tolerance-check
+      context: tolerance-check
+      docker:
+        buildArgs:
+          STOP_FAILING_TIME: '{{.STOP_FAILING_TIME}}'
+deploy:
+  statusCheckDeadlineSeconds: 120
+  kubectl:
+    manifests:
+      - tolerance-check/kubernetes/*

integration/testdata/status-check-tolerance/tolerance-check/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:latest
+ARG STOP_FAILING_TIME
+ENV STOP_FAILING_TIME=${STOP_FAILING_TIME:-not_found}  
+
+
+COPY script.sh /script.sh
+ENTRYPOINT [ "/script.sh" ]

integration/testdata/status-check-tolerance/tolerance-check/kubernetes/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tolerance-check
+  labels:
+    app: tolerance-check
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: tolerance-check
+  template:
+    metadata:
+      labels:
+        app: tolerance-check
+    spec:
+      containers:
+        - name: tolerance-check
+          image: tolerance-check

integration/testdata/status-check-tolerance/tolerance-check/script.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+current_time=$(date +%s)
+stop_failing_time=$STOP_FAILING_TIME
+
+echo $current_time
+echo "========"
+echo $stop_failing_time
+echo "========"
+
+if [[ $current_time -le $stop_failing_time ]]; then
+   echo "current time less than stop failing time, container will exit with error"
+   exit 1
+fi
+while :
+do
+	echo "Hello world!!!! - current time greater than stop failing time!"
+	sleep 2
+done

pkg/skaffold/config/options.go

@@ -32,80 +32,81 @@ type WaitForDeletions struct {
 
 // SkaffoldOptions are options that are set by command line arguments not included in the config file itself
 type SkaffoldOptions struct {
-	Apply                      bool
-	AutoBuild                  bool
-	AutoCreateConfig           bool
-	AutoDeploy                 bool
-	AutoSync                   bool
-	AssumeYes                  bool
-	CacheArtifacts             bool
-	ContainerDebugging         bool
-	Cleanup                    bool
-	DetectMinikube             bool
-	DryRun                     bool
-	EnableRPC                  bool
-	Force                      bool
-	ForceLoadImages            bool
-	IterativeStatusCheck       bool
-	FastFailStatusCheck        bool
-	Notification               bool
-	NoPrune                    bool
-	NoPruneChildren            bool
-	ProfileAutoActivation      bool
-	PropagateProfiles          bool
-	RenderOnly                 bool
-	SkipTests                  bool
-	SkipConfigDefaults         bool
-	Tail                       bool
-	WaitForConnection          bool
-	EnablePlatformNodeAffinity bool
-	EnableGKEARMNodeToleration bool
-	DisableMultiPlatformBuild  bool
-	CheckClusterNodePlatforms  bool
-	MakePathsAbsolute          *bool
-	MultiLevelRepo             *bool
-	CloudRunProject            string
-	CloudRunLocation           string
-	ConfigurationFile          string
-	HydrationDir               string
-	InventoryNamespace         string
-	InventoryID                string
-	InventoryName              string
-	GlobalConfig               string
-	EventLogFile               string
-	RenderOutput               string
-	User                       string
-	CustomTag                  string
-	Namespace                  string
-	CacheFile                  string
-	Trigger                    string
-	KubeContext                string
-	KubeConfig                 string
-	LastLogFile                string
-	DigestSource               string
-	Command                    string
-	MinikubeProfile            string
-	RepoCacheDir               string
-	TransformRulesFile         string
-	VerifyDockerNetwork        string
-	CustomLabels               []string
-	TargetImages               []string
-	Profiles                   []string
-	InsecureRegistries         []string
-	ConfigurationFilter        []string
-	HydratedManifests          []string
-	Platforms                  []string
-	BuildConcurrency           int
-	WatchPollInterval          int
-	StatusCheck                BoolOrUndefined
-	PushImages                 BoolOrUndefined
-	RPCPort                    IntOrUndefined
-	RPCHTTPPort                IntOrUndefined
-	Muted                      Muted
-	PortForward                PortForwardOptions
-	DefaultRepo                StringOrUndefined
-	SyncRemoteCache            SyncRemoteCacheOption
-	WaitForDeletions           WaitForDeletions
+	Apply                       bool
+	AutoBuild                   bool
+	AutoCreateConfig            bool
+	AutoDeploy                  bool
+	AutoSync                    bool
+	AssumeYes                   bool
+	CacheArtifacts              bool
+	ContainerDebugging          bool
+	Cleanup                     bool
+	DetectMinikube              bool
+	DryRun                      bool
+	EnableRPC                   bool
+	Force                       bool
+	ForceLoadImages             bool
+	IterativeStatusCheck        bool
+	FastFailStatusCheck         bool
+	TolerateFailuresStatusCheck bool
+	Notification                bool
+	NoPrune                     bool
+	NoPruneChildren             bool
+	ProfileAutoActivation       bool
+	PropagateProfiles           bool
+	RenderOnly                  bool
+	SkipTests                   bool
+	SkipConfigDefaults          bool
+	Tail                        bool
+	WaitForConnection           bool
+	EnablePlatformNodeAffinity  bool
+	EnableGKEARMNodeToleration  bool
+	DisableMultiPlatformBuild   bool
+	CheckClusterNodePlatforms   bool
+	MakePathsAbsolute           *bool
+	MultiLevelRepo              *bool
+	CloudRunProject             string
+	CloudRunLocation            string
+	ConfigurationFile           string
+	HydrationDir                string
+	InventoryNamespace          string
+	InventoryID                 string
+	InventoryName               string
+	GlobalConfig                string
+	EventLogFile                string
+	RenderOutput                string
+	User                        string
+	CustomTag                   string
+	Namespace                   string
+	CacheFile                   string
+	Trigger                     string
+	KubeContext                 string
+	KubeConfig                  string
+	LastLogFile                 string
+	DigestSource                string
+	Command                     string
+	MinikubeProfile             string
+	RepoCacheDir                string
+	TransformRulesFile          string
+	VerifyDockerNetwork         string
+	CustomLabels                []string
+	TargetImages                []string
+	Profiles                    []string
+	InsecureRegistries          []string
+	ConfigurationFilter         []string
+	HydratedManifests           []string
+	Platforms                   []string
+	BuildConcurrency            int
+	WatchPollInterval           int
+	StatusCheck                 BoolOrUndefined
+	PushImages                  BoolOrUndefined
+	RPCPort                     IntOrUndefined
+	RPCHTTPPort                 IntOrUndefined
+	Muted                       Muted
+	PortForward                 PortForwardOptions
+	DefaultRepo                 StringOrUndefined
+	SyncRemoteCache             SyncRemoteCacheOption
+	WaitForDeletions            WaitForDeletions
 }
 
 type RunMode string

pkg/skaffold/deploy/component/kubernetes/monitor_test.go

@@ -38,6 +38,8 @@ func (m mockStatusConfig) GetKubeContext() string { return "" }
 
 func (m mockStatusConfig) StatusCheckDeadlineSeconds() int { return 0 }
 
+func (m mockStatusConfig) StatusCheckTolerateFailures() bool { return false }
+
 func (m mockStatusConfig) FastFailStatusCheck() bool { return true }
 
 func (m mockStatusConfig) Muted() config.Muted { return config.Muted{} }