Make integration.WaitForPodsReady use pod Ready condition (#5308)

* disable TestDebug/buildpacks python3 and netcore tests due to #4088

Author: briandealwis

integration/debug_test.go

@@ -44,21 +44,21 @@ func TestDebug(t *testing.T) {
 			description: "kubectl",
 			dir:         "testdata/debug",
 			deployments: []string{"java"},
-			pods:        []string{"nodejs", "npm", "python3", "go", "netcore"},
+			pods:        []string{"nodejs", "npm" /*, "python3"*/, "go" /*, "netcore"*/},
 		},
 		{
 			description: "kustomize",
 			dir:         "testdata/debug",
 			args:        []string{"--profile", "kustomize"},
 			deployments: []string{"java"},
-			pods:        []string{"nodejs", "npm", "python3", "go", "netcore"},
+			pods:        []string{"nodejs", "npm" /*, "python3"*/, "go" /*, "netcore"*/},
 		},
 		{
 			description: "buildpacks",
 			dir:         "testdata/debug",
 			args:        []string{"--profile", "buildpacks"},
 			deployments: []string{"java"},
-			pods:        []string{"nodejs", "npm", "python3", "go", "netcore"},
+			pods:        []string{"nodejs", "npm" /*, "python3"*/, "go" /*, "netcore"*/},
 		},
 		{
 			description:   "helm",

integration/testdata/debug/skaffold.yaml

@@ -11,22 +11,22 @@ build:
     context: npm
   - image: skaffold-debug-nodejs
     context: nodejs
-  - image: skaffold-debug-python3
-    context: python3
+  #- image: skaffold-debug-python3
+  #  context: python3
   - image: skaffold-debug-go
     context: go
-  - image: skaffold-debug-netcore
-    context: netcore
+  #- image: skaffold-debug-netcore
+  #  context: netcore
 
 deploy:
   kubectl:
     manifests:
     - java/k8s/web.yaml
     - nodejs/k8s/pod.yaml
     - npm/k8s/pod.yaml
-    - python3/k8s/pod.yaml
+    #- python3/k8s/pod.yaml
     - go/k8s/pod.yaml
-    - netcore/k8s/pod.yaml
+    #- netcore/k8s/pod.yaml
 
 profiles:
 - name: kustomize
@@ -51,15 +51,15 @@ profiles:
       context: nodejs
       buildpacks:
         builder: "gcr.io/buildpacks/builder:v1"
-    - image: skaffold-debug-python3
-      context: python3
-      buildpacks:
-        builder: "gcr.io/buildpacks/builder:v1"
+    #- image: skaffold-debug-python3
+    #  context: python3
+    #  buildpacks:
+    #    builder: "gcr.io/buildpacks/builder:v1"
     - image: skaffold-debug-go
       context: go
       buildpacks:
         builder: "gcr.io/buildpacks/builder:v1"
-    - image: skaffold-debug-netcore
-      context: netcore
-      buildpacks:
-        builder: "gcr.io/buildpacks/builder:v1"
+    #- image: skaffold-debug-netcore
+    #  context: netcore
+    #  buildpacks:
+    #    builder: "gcr.io/buildpacks/builder:v1"

integration/util.go

@@ -169,17 +169,29 @@ func (k *NSKubernetesClient) CreateSecretFrom(ns, name string) {
 
 // WaitForPodsReady waits for a list of pods to become ready.
 func (k *NSKubernetesClient) WaitForPodsReady(podNames ...string) {
-	k.WaitForPodsInPhase(v1.PodRunning, podNames...)
+	f := func(pod *v1.Pod) bool {
+		for _, cond := range pod.Status.Conditions {
+			if cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+		return false
+	}
+	result := k.waitForPods(f, podNames...)
+	logrus.Infof("Pods marked as ready: %v", result)
 }
 
-// WaitForPodsInPhase waits for a list of pods to become ready.
+// WaitForPodsInPhase waits for a list of pods to reach the given phase.
 func (k *NSKubernetesClient) WaitForPodsInPhase(expectedPhase v1.PodPhase, podNames ...string) {
-	if len(podNames) == 0 {
-		return
+	f := func(pod *v1.Pod) bool {
+		return pod.Status.Phase == expectedPhase
 	}
+	result := k.waitForPods(f, podNames...)
+	logrus.Infof("Pods in phase %q: %v", expectedPhase, result)
+}
 
-	logrus.Infoln("Waiting for pods", podNames, "to be ready")
-
+// waitForPods waits for a list of pods to become ready.
+func (k *NSKubernetesClient) waitForPods(podReady func(*v1.Pod) bool, podNames ...string) (podsReady map[string]bool) {
 	ctx, cancelTimeout := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancelTimeout()
 
@@ -190,7 +202,14 @@ func (k *NSKubernetesClient) WaitForPodsInPhase(expectedPhase v1.PodPhase, podNa
 	}
 	defer w.Stop()
 
-	phases := map[string]v1.PodPhase{}
+	waitForAllPods := len(podNames) == 0
+	if waitForAllPods {
+		logrus.Infof("Waiting for all pods in namespace %q to be ready", k.ns)
+	} else {
+		logrus.Infoln("Waiting for pods", podNames, "to be ready")
+	}
+
+	podsReady = map[string]bool{}
 
 	for {
 	waitLoop:
@@ -200,30 +219,40 @@ func (k *NSKubernetesClient) WaitForPodsInPhase(expectedPhase v1.PodPhase, podNa
 			//k.debug("nodes")
 			k.debug("pods")
 			k.logs("pod", podNames)
-			k.t.Fatalf("Timed out waiting for pods %v ready in namespace %s", podNames, k.ns)
+			k.t.Fatalf("Timed out waiting for pods %v in namespace %q", podNames, k.ns)
 
 		case event := <-w.ResultChan():
 			if event.Object == nil {
 				return
 			}
 			pod := event.Object.(*v1.Pod)
-			logrus.Infoln("Pod", pod.Name, "is", pod.Status.Phase)
 			if pod.Status.Phase == v1.PodFailed {
 				logs, err := pods.GetLogs(pod.Name, &v1.PodLogOptions{}).DoRaw(ctx)
 				if err != nil {
 					k.t.Fatalf("failed to get logs for failed pod %s: %s", pod.Name, err)
 				}
 				k.t.Fatalf("pod %s failed. Logs:\n %s", pod.Name, logs)
 			}
-			phases[pod.Name] = pod.Status.Phase
 
+			if _, found := podsReady[pod.Name]; !found && waitForAllPods {
+				podNames = append(podNames, pod.Name)
+			}
+			podsReady[pod.Name] = podReady(pod)
+
+			var waiting []string
 			for _, podName := range podNames {
-				if phases[podName] != expectedPhase {
-					break waitLoop
+				if !podsReady[podName] {
+					waiting = append(waiting, podName)
 				}
 			}
-
-			logrus.Infoln("Pods", podNames, "ready")
+			if len(waiting) > 0 {
+				logrus.Infof("Still waiting for pods %v", waiting)
+				break waitLoop
+			} else if l := len(w.ResultChan()); l > 0 {
+				// carry on when there are pending messages in case a new pod has been created
+				logrus.Infof("%d pending pod update messages", l)
+				break waitLoop
+			}
 			return
 		}
 	}