Reduce flakiness by ensuring three successful fetches before returning.

Author: nat-henderson

google/iam.go

@@ -64,6 +64,26 @@ func iamPolicyReadModifyWrite(updater ResourceIamUpdater, modify iamPolicyModify
 		log.Printf("[DEBUG]: Setting policy for %s to %+v\n", updater.DescribeResource(), p)
 		err = updater.SetResourceIamPolicy(p)
 		if err == nil {
+			for successfulFetches := 0; successfulFetches < 3; {
+				new_p, err := updater.GetResourceIamPolicy()
+				if err != nil {
+					return err
+				}
+				modified_p := new_p
+				err = modify(modified_p)
+				if err != nil {
+					return err
+				}
+				if modified_p == new_p {
+					successfulFetches += 1
+				} else {
+					backoff = backoff * 2
+					if backoff > 30*time.Second {
+						return fmt.Errorf("Error applying IAM policy to %s: Waited too long for propagation.\n", updater.DescribeResource())
+					}
+				}
+				time.Sleep(backoff * 5)
+			}
 			break
 		}
 		if isConflictError(err) {