Merge branch 'main' into MemorystoreInstanceReplication

Author: NA2047

.ci/infra/terraform/main.tf

@@ -321,6 +321,7 @@ module "project-services" {
     "parametermanager.googleapis.com",
     "privateca.googleapis.com",
     "privilegedaccessmanager.googleapis.com",
+    "progressiverollout.googleapis.com",
     "pubsub.googleapis.com",
     "pubsublite.googleapis.com",
     "publicca.googleapis.com",
@@ -393,6 +394,22 @@ resource "google_project_service_identity" "sqladmin_sa" {
   service = "sqladmin.googleapis.com"
 }
 
+resource "google_project_service_identity" "osconfig_sa" {
+  provider = google-beta
+  depends_on = [module.project-services]
+
+  project = google_project.proj.project_id
+  service = "osconfig.googleapis.com"
+}
+
+resource "google_project_service_identity" "progressiverollout_sa" {
+  provider = google-beta
+  depends_on = [module.project-services]
+
+  project = google_project.proj.project_id
+  service = "progressiverollout.googleapis.com"
+}
+
 # TestAccComposerEnvironment_fixPyPiPackages
 # TestAccComposerEnvironmentComposer2_private
 # TestAccComposerEnvironment_withEncryptionConfigComposer1

.ci/magician/github/get.go

@@ -51,7 +51,7 @@ func (gh *Client) GetPullRequest(prNumber string) (PullRequest, error) {
 
 	var pullRequest PullRequest
 
-	err := utils.RequestCall(url, "GET", gh.token, &pullRequest, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &pullRequest, nil)
 
 	return pullRequest, err
 }
@@ -61,7 +61,7 @@ func (gh *Client) GetPullRequests(state, base, sort, direction string) ([]PullRe
 
 	var pullRequests []PullRequest
 
-	err := utils.RequestCall(url, "GET", gh.token, &pullRequests, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &pullRequests, nil)
 
 	return pullRequests, err
 }
@@ -73,7 +73,7 @@ func (gh *Client) GetPullRequestRequestedReviewers(prNumber string) ([]User, err
 		Users []User `json:"users"`
 	}
 
-	err := utils.RequestCall(url, "GET", gh.token, &requestedReviewers, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &requestedReviewers, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -88,7 +88,7 @@ func (gh *Client) GetPullRequestPreviousReviewers(prNumber string) ([]User, erro
 		User User `json:"user"`
 	}
 
-	err := utils.RequestCall(url, "GET", gh.token, &reviews, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &reviews, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -110,7 +110,7 @@ func (gh *Client) GetPullRequestComments(prNumber string) ([]PullRequestComment,
 	url := fmt.Sprintf("https://api.github.com/repos/GoogleCloudPlatform/magic-modules/issues/%s/comments", prNumber)
 
 	var comments []PullRequestComment
-	err := utils.RequestCall(url, "GET", gh.token, &comments, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &comments, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +121,7 @@ func (gh *Client) GetTeamMembers(organization, team string) ([]User, error) {
 	url := fmt.Sprintf("https://api.github.com/orgs/%s/teams/%s/members", organization, team)
 
 	var members []User
-	err := utils.RequestCall(url, "GET", gh.token, &members, nil)
+	err := utils.RequestCallWithRetry(url, "GET", gh.token, &members, nil)
 	if err != nil {
 		return nil, err
 	}

.ci/magician/github/membership.go

@@ -76,7 +76,7 @@ func IsCoreReviewer(user string) bool {
 
 func isOrgMember(author, org, githubToken string) bool {
 	url := fmt.Sprintf("https://api.github.com/orgs/%s/members/%s", org, author)
-	err := utils.RequestCall(url, "GET", githubToken, nil, nil)
+	err := utils.RequestCallWithRetry(url, "GET", githubToken, nil, nil)
 
 	return err == nil
 }

.ci/magician/github/set.go

@@ -18,6 +18,7 @@ package github
 import (
 	"fmt"
 	utils "magician/utility"
+	"strings"
 )
 
 func (gh *Client) PostBuildStatus(prNumber, title, state, targetURL, commitSha string) error {
@@ -29,7 +30,7 @@ func (gh *Client) PostBuildStatus(prNumber, title, state, targetURL, commitSha s
 		"target_url": targetURL,
 	}
 
-	err := utils.RequestCall(url, "POST", gh.token, nil, postBody)
+	err := utils.RequestCallWithRetry(url, "POST", gh.token, nil, postBody)
 	if err != nil {
 		return err
 	}
@@ -46,7 +47,7 @@ func (gh *Client) PostComment(prNumber, comment string) error {
 		"body": comment,
 	}
 
-	err := utils.RequestCall(url, "POST", gh.token, nil, body)
+	err := utils.RequestCallWithRetry(url, "POST", gh.token, nil, body)
 	if err != nil {
 		return err
 	}
@@ -63,7 +64,7 @@ func (gh *Client) UpdateComment(prNumber, comment string, id int) error {
 		"body": comment,
 	}
 
-	err := utils.RequestCall(url, "PATCH", gh.token, nil, body)
+	err := utils.RequestCallWithRetry(url, "PATCH", gh.token, nil, body)
 	if err != nil {
 		return err
 	}
@@ -81,7 +82,7 @@ func (gh *Client) RequestPullRequestReviewers(prNumber string, reviewers []strin
 		"team_reviewers": {},
 	}
 
-	err := utils.RequestCall(url, "POST", gh.token, nil, body)
+	err := utils.RequestCallWithRetry(url, "POST", gh.token, nil, body)
 	if err != nil {
 		return err
 	}
@@ -97,7 +98,7 @@ func (gh *Client) AddLabels(prNumber string, labels []string) error {
 	body := map[string][]string{
 		"labels": labels,
 	}
-	err := utils.RequestCall(url, "POST", gh.token, nil, body)
+	err := utils.RequestCallWithRetry(url, "POST", gh.token, nil, body)
 
 	if err != nil {
 		return fmt.Errorf("failed to add %q labels: %s", labels, err)
@@ -109,7 +110,7 @@ func (gh *Client) AddLabels(prNumber string, labels []string) error {
 
 func (gh *Client) RemoveLabel(prNumber, label string) error {
 	url := fmt.Sprintf("https://api.github.com/repos/GoogleCloudPlatform/magic-modules/issues/%s/labels/%s", prNumber, label)
-	err := utils.RequestCall(url, "DELETE", gh.token, nil, nil)
+	err := utils.RequestCallWithRetry(url, "DELETE", gh.token, nil, nil)
 
 	if err != nil {
 		return fmt.Errorf("failed to remove %s label: %s", label, err)
@@ -120,7 +121,7 @@ func (gh *Client) RemoveLabel(prNumber, label string) error {
 
 func (gh *Client) CreateWorkflowDispatchEvent(workflowFileName string, inputs map[string]any) error {
 	url := fmt.Sprintf("https://api.github.com/repos/GoogleCloudPlatform/magic-modules/actions/workflows/%s/dispatches", workflowFileName)
-	err := utils.RequestCall(url, "POST", gh.token, nil, map[string]any{
+	err := utils.RequestCallWithRetry(url, "POST", gh.token, nil, map[string]any{
 		"ref":    "main",
 		"inputs": inputs,
 	})
@@ -136,16 +137,21 @@ func (gh *Client) CreateWorkflowDispatchEvent(workflowFileName string, inputs ma
 
 func (gh *Client) MergePullRequest(owner, repo, prNumber, commitSha string) error {
 	url := fmt.Sprintf("https://api.github.com/repos/%s/%s/pulls/%s/merge", owner, repo, prNumber)
-	err := utils.RequestCall(url, "PUT", gh.token, nil, map[string]any{
+
+	err := utils.RequestCallWithRetry(url, "PUT", gh.token, nil, map[string]any{
 		"merge_method": "squash",
 		"sha":          commitSha,
 	})
 
 	if err != nil {
+		// Check if the error is "Merge already in progress" (405)
+		if strings.Contains(err.Error(), "Merge already in progress") {
+			fmt.Printf("Pull request %s is already being merged\n", prNumber)
+			return nil
+		}
 		return fmt.Errorf("failed to merge pull request: %s", err)
 	}
 
 	fmt.Printf("Successfully merged pull request %s\n", prNumber)
-
 	return nil
 }

.ci/magician/utility/utils.go

@@ -20,57 +20,167 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"os"
+	"time"
 
 	"golang.org/x/exp/slices"
 )
 
-func RequestCall(url, method, credentials string, result any, body any) error {
+// retryConfig holds configuration for request retries
+type retryConfig struct {
+	MaxRetries       int           // Maximum number of retry attempts
+	InitialBackoff   time.Duration // Initial backoff duration
+	MaxBackoff       time.Duration // Maximum backoff duration
+	BackoffFactor    float64       // Factor by which to multiply backoff on each retry
+	RetryStatusCodes []int         // HTTP status codes that should trigger a retry
+}
+
+// defaultRetryConfig provides default retry configuration
+func defaultRetryConfig() retryConfig {
+	return retryConfig{
+		MaxRetries:       3,
+		InitialBackoff:   5000 * time.Millisecond,
+		MaxBackoff:       60 * time.Second,
+		BackoffFactor:    2.0,
+		RetryStatusCodes: []int{408, 429, 500, 502, 503, 504}, // Common retry status codes
+	}
+}
+
+// makeHTTPRequest performs the actual HTTP request and returns the response
+func makeHTTPRequest(url, method, credentials string, body any) (*http.Response, []byte, error) {
 	client := &http.Client{}
 	jsonBody, err := json.Marshal(body)
 	if err != nil {
-		return fmt.Errorf("error marshaling JSON: %s", err)
+		return nil, nil, fmt.Errorf("error marshaling JSON: %s", err)
 	}
 	req, err := http.NewRequest(method, url, bytes.NewBuffer(jsonBody))
 	if err != nil {
-		return fmt.Errorf("error creating request: %s", err)
+		return nil, nil, fmt.Errorf("error creating request: %s", err)
 	}
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", credentials))
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	fmt.Println("")
 	fmt.Println("request url: ", url)
-	fmt.Println("request body: ", string(jsonBody)) // Convert to string
+	fmt.Println("request body: ", string(jsonBody))
 	fmt.Println("")
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	defer resp.Body.Close()
 
 	respBodyBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	fmt.Println("response status-code: ", resp.StatusCode)
-	fmt.Println("response body: ", string(respBodyBytes)) // Convert to string
+	fmt.Println("response body: ", string(respBodyBytes))
 	fmt.Println("")
 
-	// Decode the response, if needed
+	return resp, respBodyBytes, nil
+}
+
+// processResponse handles the response and unmarshals it to the result if provided
+func processResponse(resp *http.Response, respBodyBytes []byte, result any) error {
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		var errorResponse struct {
+			Message string `json:"message"`
+			Error   string `json:"error"`
+		}
+
+		if err := json.Unmarshal(respBodyBytes, &errorResponse); err == nil {
+			errorMsg := errorResponse.Message
+			if errorMsg == "" {
+				errorMsg = errorResponse.Error
+			}
+
+			if errorMsg != "" {
+				return fmt.Errorf("got code %d from server: %s", resp.StatusCode, errorMsg)
+			}
+		}
+
+		// Fall back to generic error if we couldn't parse the error message
+		return fmt.Errorf("got code %d from server", resp.StatusCode)
+	}
+
+	// If no error status code, decode the response if needed
 	if result != nil {
-		if err = json.Unmarshal(respBodyBytes, &result); err != nil {
+		if err := json.Unmarshal(respBodyBytes, &result); err != nil {
 			return err
 		}
 	}
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("got code %d from server", resp.StatusCode)
+	return nil
+}
+
+// RequestCall makes a single HTTP request without retries
+func RequestCall(url, method, credentials string, result any, body any) error {
+	resp, respBodyBytes, err := makeHTTPRequest(url, method, credentials, body)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	return processResponse(resp, respBodyBytes, result)
+}
+
+// shouldRetry determines if a retry should be attempted based on the status code
+func shouldRetry(statusCode int, retryConfig retryConfig) bool {
+	return slices.Contains(retryConfig.RetryStatusCodes, statusCode)
+}
+
+// calculateBackoff calculates the backoff duration for the current retry attempt
+func calculateBackoff(attempt int, config retryConfig) time.Duration {
+	backoff := config.InitialBackoff * time.Duration(math.Pow(config.BackoffFactor, float64(attempt)))
+	if backoff > config.MaxBackoff {
+		backoff = config.MaxBackoff
+	}
+	return backoff
+}
+
+// RequestCallWithRetry makes an HTTP request with retry capability
+func requestCallWithRetry(url, method, credentials string, result any, body any, config retryConfig) error {
+	var lastErr error
+
+	for attempt := 0; attempt <= config.MaxRetries; attempt++ {
+		// If this is a retry attempt, wait before trying again
+		if attempt > 0 {
+			backoff := calculateBackoff(attempt-1, config)
+			fmt.Printf("Retry attempt %d after %v\n", attempt, backoff)
+			time.Sleep(backoff)
+		}
+
+		resp, respBodyBytes, err := makeHTTPRequest(url, method, credentials, body)
+		if err != nil {
+			lastErr = err
+			continue // Network error, retry
+		}
+
+		// Process the response
+		err = processResponse(resp, respBodyBytes, result)
+		if err != nil {
+			lastErr = err
+
+			// Check if we should retry based on status code
+			if shouldRetry(resp.StatusCode, config) {
+				continue
+			}
+		}
+
+		// If we got here with no error, return success
+		return err
+	}
+
+	return fmt.Errorf("max retries exceeded: %w", lastErr)
+}
+
+// RequestCallWithRetry is a convenience function that uses default retry settings
+func RequestCallWithRetry(url, method, credentials string, result any, body any) error {
+	return requestCallWithRetry(url, method, credentials, result, body, defaultRetryConfig())
 }
 
 func Removes(s1 []string, s2 []string) []string {