retry on resourceNotReady (#4433)

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

google/common_operation.go

@@ -18,6 +18,9 @@ type Waiter interface {
 	// if the operation has no current error.
 	Error() error
 
+	// IsRetryable returns whether a given error should be retried.
+	IsRetryable(error) bool
+
 	// SetOp sets the operation we're waiting on in a Waiter struct so that it
 	// can be used in other methods.
 	SetOp(interface{}) error
@@ -59,6 +62,10 @@ func (w *CommonOperationWaiter) Error() error {
 	return nil
 }
 
+func (w *CommonOperationWaiter) IsRetryable(error) bool {
+	return false
+}
+
 func (w *CommonOperationWaiter) SetOp(op interface{}) error {
 	if err := Convert(op, &w.Op); err != nil {
 		return err
@@ -110,6 +117,10 @@ func CommonRefreshFunc(w Waiter) resource.StateRefreshFunc {
 		}
 
 		if err = w.Error(); err != nil {
+			if w.IsRetryable(err) {
+				log.Printf("[DEBUG] Retrying operation GET based on retryable err: %s", err)
+				return op, w.State(), nil
+			}
 			return nil, "", err
 		}
 

google/compute_operation.go

@@ -29,6 +29,17 @@ func (w *ComputeOperationWaiter) Error() error {
 	return nil
 }
 
+func (w *ComputeOperationWaiter) IsRetryable(err error) bool {
+	if oe, ok := err.(ComputeOperationError); ok {
+		for _, e := range oe.Errors {
+			if e.Code == "RESOURCE_NOT_READY" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func (w *ComputeOperationWaiter) SetOp(op interface{}) error {
 	var ok bool
 	w.Op, ok = op.(*compute.Operation)

google/container_operation.go

@@ -41,6 +41,10 @@ func (w *ContainerOperationWaiter) Error() error {
 	return nil
 }
 
+func (w *ContainerOperationWaiter) IsRetryable(error) bool {
+	return false
+}
+
 func (w *ContainerOperationWaiter) SetOp(op interface{}) error {
 	var ok bool
 	w.Op, ok = op.(*container.Operation)

google/dataproc_job_operation.go

@@ -28,6 +28,10 @@ func (w *DataprocJobOperationWaiter) Error() error {
 	return nil
 }
 
+func (w *DataprocJobOperationWaiter) IsRetryable(error) bool {
+	return false
+}
+
 func (w *DataprocJobOperationWaiter) SetOp(job interface{}) error {
 	// The "operation" is just the job. Instead of holding onto the whole job
 	// object, we only care about the state, which gets set in QueryOp, so this

google/sqladmin_operation.go

@@ -33,6 +33,10 @@ func (w *SqlAdminOperationWaiter) Error() error {
 	return nil
 }
 
+func (w *SqlAdminOperationWaiter) IsRetryable(error) bool {
+	return false
+}
+
 func (w *SqlAdminOperationWaiter) SetOp(op interface{}) error {
 	if op == nil {
 		// Starting as a log statement, this may be a useful error in the future