If webhook deployment fails, upload logs

deploy/webhook/deployment.yaml

@@ -37,11 +37,20 @@ spec:
       - name: docs-controller
         image: gcr.io/k8s-skaffold/docs-controller:latest
         args: ["/webhook"]
+        volumeMounts:
+        - name: webhook
+          mountPath: /secret
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:
             secretKeyRef:
               name: github-token 
               key: token
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /secret/webhook.json
         ports:
         - containerPort: 8080
+      volumes:
+        - name: webhook
+          secret:
+            secretName: webhook

pkg/webhook/constants/constants.go

@@ -46,4 +46,7 @@ const (
 
 	// DeploymentImage is the image the controller deploys, must contain hugo and git
 	DeploymentImage = "gcr.io/k8s-skaffold/docs-controller:latest"
+
+	// LogsGCSBucket is the GCS bucket logs are uploaded to
+	LogsGCSBucket = "webhook-logs"
 )

pkg/webhook/gcs/gcs.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2018 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	cstorage "cloud.google.com/go/storage"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/webhook/constants"
+	"github.com/GoogleContainerTools/skaffold/pkg/webhook/kubernetes"
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+)
+
+// UploadDeploymentLogsToBucket gets logs from d and uploads them to the bucket
+func UploadDeploymentLogsToBucket(d *appsv1.Deployment, prNumber int) (string, error) {
+	logs := kubernetes.Logs(d)
+	ctx := context.Background()
+	c, err := cstorage.NewClient(ctx)
+	if err != nil {
+		return "", errors.Wrap(err, "creating GCS client")
+	}
+	defer c.Close()
+	name := fmt.Sprintf("logs-%d-%d", prNumber, time.Now().UnixNano())
+	w := c.Bucket(constants.LogsGCSBucket).Object(name).NewWriter(ctx)
+	defer w.Close()
+	if _, err := io.Copy(w, bytes.NewBuffer([]byte(logs))); err != nil {
+		return "", err
+	}
+	return name, nil
+}

pkg/webhook/kubernetes/deployment.go

@@ -19,6 +19,9 @@ package kubernetes
 import (
 	"context"
 	"fmt"
+	"log"
+	"net/http"
+	"os/exec"
 	"path"
 	"time"
 
@@ -30,11 +33,13 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
-	emptyVol     = "empty-vol"
-	emptyVolPath = "/empty"
+	initContainerName = "git-clone"
+	emptyVol          = "empty-vol"
+	emptyVolPath      = "/empty"
 )
 
 // CreateDeployment creates a deployment for this pull request
@@ -71,7 +76,7 @@ func CreateDeployment(pr *github.PullRequestEvent, svc *v1.Service, externalIP s
 				Spec: v1.PodSpec{
 					InitContainers: []v1.Container{
 						{
-							Name:       "git-clone",
+							Name:       initContainerName,
 							Image:      constants.DeploymentImage,
 							Args:       []string{"git", "clone", userRepo, "--branch", pr.PullRequest.Head.GetRef()},
 							WorkingDir: emptyVolPath,
@@ -118,14 +123,45 @@ func CreateDeployment(pr *github.PullRequestEvent, svc *v1.Service, externalIP s
 }
 
 // WaitForDeploymentToStabilize waits till the Deployment has stabilized
-func WaitForDeploymentToStabilize(d *appsv1.Deployment) error {
+func WaitForDeploymentToStabilize(d *appsv1.Deployment, ip string) error {
 	client, err := pkgkubernetes.GetClientset()
 	if err != nil {
 		return errors.Wrap(err, "getting clientset")
 	}
-	return pkgkubernetes.WaitForDeploymentToStabilize(context.Background(), client, d.Namespace, d.Name, 5*time.Minute)
+	if err := pkgkubernetes.WaitForDeploymentToStabilize(context.Background(), client, d.Namespace, d.Name, 5*time.Minute); err != nil {
+		return errors.Wrap(err, "waiting for deployment to stabilize")
+	}
+	// wait up to five minutes for the URL to return a valid endpoint
+	url := BaseURL(ip)
+	log.Printf("Waiting up to 5 minutes for %s to return an OK response...", url)
+	return wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return false, nil
+		}
+		return resp.StatusCode == http.StatusOK, nil
+	})
 }
 
+// BaseURL returns the base url of the deployment
 func BaseURL(ip string) string {
 	return fmt.Sprintf("http://%s:%d", ip, constants.HugoPort)
 }
+
+// Logs returns the logs for both containers for the given deployment
+func Logs(d *appsv1.Deployment) string {
+	deploy := fmt.Sprintf("deployment/%s", d.Name)
+	// get init container logs
+	cmd := exec.Command("kubectl", "logs", deploy, "-c", initContainerName)
+	initLogs, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("Error retrieving init container logs for %s: %v", d.Name, err)
+	}
+	// get deployment logs
+	cmd = exec.Command("kubectl", "logs", deploy)
+	logs, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("Error retrieving deployment logs for %s: %v", d.Name, err)
+	}
+	return fmt.Sprintf("Init container logs: \n %s \nContainer Logs: \n %s", initLogs, logs)
+}

webhook/webhook.go

@@ -22,13 +22,15 @@ import (
 	"log"
 	"net/http"
 
+	"github.com/GoogleContainerTools/skaffold/pkg/webhook/gcs"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/webhook/constants"
+	pkggithub "github.com/GoogleContainerTools/skaffold/pkg/webhook/github"
 	"github.com/GoogleContainerTools/skaffold/pkg/webhook/kubernetes"
 	"github.com/GoogleContainerTools/skaffold/pkg/webhook/labels"
 	"github.com/google/go-github/github"
 	"github.com/pkg/errors"
-
-	"github.com/GoogleContainerTools/skaffold/pkg/webhook/constants"
-	pkggithub "github.com/GoogleContainerTools/skaffold/pkg/webhook/github"
+	appsv1 "k8s.io/api/apps/v1"
 )
 
 const (
@@ -54,6 +56,7 @@ func handleGithubEvent(w http.ResponseWriter, r *http.Request) {
 		log.Printf("error decoding pr event: %v", err)
 	}
 	if err := handlePullRequestEvent(event); err != nil {
+		commentOnGithub(event, "Error creating deployment, please see controller logs for details.")
 		log.Printf("error handling pr event: %v", err)
 	}
 }
@@ -104,20 +107,45 @@ func handlePullRequestEvent(event *github.PullRequestEvent) error {
 	if err != nil {
 		return errors.Wrapf(err, "creating deployment for PR %d", prNumber)
 	}
-	if err := kubernetes.WaitForDeploymentToStabilize(deployment); err != nil {
-		return errors.Wrapf(err, "waiting for deployment %s to stabilize", deployment.Name)
+	response := succeeded
+	if err := kubernetes.WaitForDeploymentToStabilize(deployment, ip); err != nil {
+		log.Printf("Deployment didn't stabilize, commenting with failure message...")
+		response = failed
 	}
 
-	// Comment on the PR and remove the docs-modifications label
-	githubClient := pkggithub.NewClient()
+	msg, err := response(deployment, event, ip)
+	if err != nil {
+		return errors.Wrapf(err, "getting github message")
+	}
+
+	if err := commentOnGithub(event, msg); err != nil {
+		return errors.Wrap(err, "commenting on github")
+	}
+
+	return nil
+}
+
+func succeeded(d *appsv1.Deployment, event *github.PullRequestEvent, ip string) (string, error) {
 	baseURL := kubernetes.BaseURL(ip)
-	msg := fmt.Sprintf("Please visit [%s](%s) to view changes to the docs.", baseURL, baseURL)
+	return fmt.Sprintf("Please visit [%s](%s) to view changes to the docs.", baseURL, baseURL), nil
+}
+
+func failed(d *appsv1.Deployment, event *github.PullRequestEvent, ip string) (string, error) {
+	name, err := gcs.UploadDeploymentLogsToBucket(d, event.GetNumber())
+	if err != nil {
+		return "", errors.Wrapf(err, "uploading logs to bucket")
+	}
+	url := fmt.Sprintf("https://storage.googleapis.com/%s/%s", constants.LogsGCSBucket, name)
+	return fmt.Sprintf("Error creating deployment %s, please visit %s to view logs.", d.Name, url), nil
+}
+
+func commentOnGithub(event *github.PullRequestEvent, msg string) error {
+	githubClient := pkggithub.NewClient()
 	if err := githubClient.CommentOnPR(event, msg); err != nil {
-		return errors.Wrapf(err, "comenting on PR %d", prNumber)
+		return errors.Wrapf(err, "comenting on PR %d", event.GetNumber())
 	}
 	if err := githubClient.RemoveLabelFromPR(event, constants.DocsLabel); err != nil {
-		return errors.Wrapf(err, "removing %s label from PR %d", constants.DocsLabel, prNumber)
+		return errors.Wrapf(err, "removing %s label from PR %d", constants.DocsLabel, event.GetNumber())
 	}
-
 	return nil
 }