Retry instance metadata on fingerprint mismatch. (#3372)

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

google/metadata.go

@@ -3,37 +3,24 @@ package google
 import (
 	"errors"
 	"fmt"
-	"strings"
-
 	computeBeta "google.golang.org/api/compute/v0.beta"
 	"google.golang.org/api/compute/v1"
 )
 
-const FINGERPRINT_RETRIES = 10
-
-var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}
+const METADATA_FINGERPRINT_RETRIES = 10
 
 // Since the google compute API uses optimistic locking, there is a chance
 // we need to resubmit our updated metadata. To do this, you need to provide
 // an update function that attempts to submit your metadata
 func MetadataRetryWrapper(update func() error) error {
 	attempt := 0
-	for attempt < FINGERPRINT_RETRIES {
+	for attempt < METADATA_FINGERPRINT_RETRIES {
 		err := update()
 		if err == nil {
 			return nil
 		}
 
-		// Check to see if the error matches any of our fingerprint-related failure messages
-		var fingerprintError bool
-		for _, msg := range FINGERPRINT_FAIL_ERRORS {
-			if strings.Contains(err.Error(), msg) {
-				fingerprintError = true
-				break
-			}
-		}
-
-		if !fingerprintError {
+		if !isFingerprintError(err) {
 			// Something else went wrong, don't retry
 			return err
 		}

google/resource_compute_instance.go

@@ -933,14 +933,34 @@ func resourceComputeInstanceUpdate(d *schema.ResourceData, meta interface{}) err
 			return err
 		}
 
-		op, err := config.clientCompute.Instances.SetMetadata(project, zone, d.Id(), metadataV1).Do()
-		if err != nil {
-			return fmt.Errorf("Error updating metadata: %s", err)
-		}
+		// We're retrying for an error 412 where the metadata fingerprint is out of date
+		err = retry(
+			func() error {
+				// retrieve up-to-date metadata from the API in case several updates hit simultaneously. instances
+				// sometimes but not always share metadata fingerprints.
+				instance, err := config.clientComputeBeta.Instances.Get(project, zone, d.Id()).Do()
+				if err != nil {
+					return fmt.Errorf("Error retrieving metadata: %s", err)
+				}
 
-		opErr := computeOperationWaitTime(config.clientCompute, op, project, "metadata to update", int(d.Timeout(schema.TimeoutUpdate).Minutes()))
-		if opErr != nil {
-			return opErr
+				metadataV1.Fingerprint = instance.Metadata.Fingerprint
+
+				op, err := config.clientCompute.Instances.SetMetadata(project, zone, d.Id(), metadataV1).Do()
+				if err != nil {
+					return fmt.Errorf("Error updating metadata: %s", err)
+				}
+
+				opErr := computeOperationWaitTime(config.clientCompute, op, project, "metadata to update", int(d.Timeout(schema.TimeoutUpdate).Minutes()))
+				if opErr != nil {
+					return opErr
+				}
+
+				return nil
+			},
+		)
+
+		if err != nil {
+			return err
 		}
 
 		d.SetPartial("metadata")

google/utils.go

@@ -125,6 +125,20 @@ func isFailedPreconditionError(err error) bool {
 	return false
 }
 
+var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}
+
+// We've encountered a few common fingerprint-related strings; if this is one of
+// them, we're confident this is an error due to fingerprints.
+func isFingerprintError(err error) bool {
+	for _, msg := range FINGERPRINT_FAIL_ERRORS {
+		if strings.Contains(err.Error(), msg) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func isConflictError(err error) bool {
 	if e, ok := err.(*googleapi.Error); ok && e.Code == 409 {
 		return true
@@ -369,6 +383,11 @@ func isRetryableError(err error) bool {
 		return true
 	}
 
+	if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 412) && isFingerprintError(err) {
+		log.Printf("[DEBUG] Dismissed an error as retryable as a fingerprint mismatch: %s", err)
+		return true
+	}
+
 	return false
 }