Fix issue #12883 (#8738) (#15828)

* Fix issue #12883

Make one CloudResourceManager call per monitored_project terraform
resource rather than per (monitored_project * "sibling" projects)

* Add comment

* Correct build errors

* Fix build errors

* Add debug logs

* Fix printf

* Fix match between TF resource and API response

* Fix from non-matching to matching condition

* Correct documentation

* Correct documentation

* Fix go build issues

* Convert projectNumber to string

* Use correct string conversion

* Move comment

* Add debug logs to monitored project encoder

* correct delete_url

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/8738.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+monitoring: fixed scaling issues when deploying terraform changes with many `google_monitoring_monitored_project`
+```

google/services/monitoring/resource_monitoring_monitored_project.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"log"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
@@ -116,7 +117,7 @@ func resourceMonitoringMonitoredProjectCreate(d *schema.ResourceData, meta inter
 	}
 
 	obj := make(map[string]interface{})
-	nameProp, err := expandNestedMonitoringMonitoredProjectName(d.Get("name"), d, config)
+	nameProp, err := expandMonitoringMonitoredProjectName(d.Get("name"), d, config)
 	if err != nil {
 		return err
 	} else if v, ok := d.GetOkExists("name"); !tpgresource.IsEmptyValue(reflect.ValueOf(nameProp)) && (ok || !reflect.DeepEqual(v, nameProp)) {
@@ -208,18 +209,6 @@ func resourceMonitoringMonitoredProjectRead(d *schema.ResourceData, meta interfa
 		return transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("MonitoringMonitoredProject %q", d.Id()))
 	}
 
-	res, err = flattenNestedMonitoringMonitoredProject(d, meta, res)
-	if err != nil {
-		return err
-	}
-
-	if res == nil {
-		// Object isn't there any more - remove it from the state.
-		log.Printf("[DEBUG] Removing MonitoringMonitoredProject because it couldn't be matched.")
-		d.SetId("")
-		return nil
-	}
-
 	res, err = resourceMonitoringMonitoredProjectDecoder(d, meta, res)
 	if err != nil {
 		return err
@@ -232,10 +221,10 @@ func resourceMonitoringMonitoredProjectRead(d *schema.ResourceData, meta interfa
 		return nil
 	}
 
-	if err := d.Set("name", flattenNestedMonitoringMonitoredProjectName(res["name"], d, config)); err != nil {
+	if err := d.Set("name", flattenMonitoringMonitoredProjectName(res["name"], d, config)); err != nil {
 		return fmt.Errorf("Error reading MonitoredProject: %s", err)
 	}
-	if err := d.Set("create_time", flattenNestedMonitoringMonitoredProjectCreateTime(res["createTime"], d, config)); err != nil {
+	if err := d.Set("create_time", flattenMonitoringMonitoredProjectCreateTime(res["createTime"], d, config)); err != nil {
 		return fmt.Errorf("Error reading MonitoredProject: %s", err)
 	}
 
@@ -251,7 +240,7 @@ func resourceMonitoringMonitoredProjectDelete(d *schema.ResourceData, meta inter
 
 	billingProject := ""
 
-	url, err := tpgresource.ReplaceVars(d, config, "{{MonitoringBasePath}}v1/locations/global/metricsScopes/{{metrics_scope}}/projects/{{name}}")
+	url, err := tpgresource.ReplaceVars(d, config, "{{MonitoringBasePath}}v1/{{name}}")
 	if err != nil {
 		return err
 	}
@@ -308,107 +297,70 @@ func resourceMonitoringMonitoredProjectImport(d *schema.ResourceData, meta inter
 	return []*schema.ResourceData{d}, nil
 }
 
-func flattenNestedMonitoringMonitoredProjectName(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+func flattenMonitoringMonitoredProjectName(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
 	return v
 }
 
-func flattenNestedMonitoringMonitoredProjectCreateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+func flattenMonitoringMonitoredProjectCreateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
 	return v
 }
 
-func expandNestedMonitoringMonitoredProjectName(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+func expandMonitoringMonitoredProjectName(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
 	return v, nil
 }
 
 func resourceMonitoringMonitoredProjectEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) {
 	name := d.Get("name").(string)
+	log.Printf("[DEBUG] Encoded monitored project name: %s", name)
 	name = tpgresource.GetResourceNameFromSelfLink(name)
+	log.Printf("[DEBUG] Encoded monitored project resource name: %s", name)
 	d.Set("name", name)
 	metricsScope := d.Get("metrics_scope").(string)
+	log.Printf("[DEBUG] Encoded monitored project metricsScope: %s", metricsScope)
 	metricsScope = tpgresource.GetResourceNameFromSelfLink(metricsScope)
+	log.Printf("[DEBUG] Encoded monitored project metricsScope resource name: %s", metricsScope)
 	d.Set("metrics_scope", metricsScope)
 	obj["name"] = fmt.Sprintf("locations/global/metricsScopes/%s/projects/%s", metricsScope, name)
 	return obj, nil
 }
 
-func flattenNestedMonitoringMonitoredProject(d *schema.ResourceData, meta interface{}, res map[string]interface{}) (map[string]interface{}, error) {
-	var v interface{}
-	var ok bool
-
-	v, ok = res["monitoredProjects"]
-	if !ok || v == nil {
-		return nil, nil
-	}
-
-	switch v.(type) {
-	case []interface{}:
-		break
-	case map[string]interface{}:
-		// Construct list out of single nested resource
-		v = []interface{}{v}
-	default:
-		return nil, fmt.Errorf("expected list or map for value monitoredProjects. Actual value: %v", v)
-	}
+func resourceMonitoringMonitoredProjectDecoder(d *schema.ResourceData, meta interface{}, res map[string]interface{}) (map[string]interface{}, error) {
+	// terraform resource config
+	config := meta.(*transport_tpg.Config)
 
-	_, item, err := resourceMonitoringMonitoredProjectFindNestedObjectInList(d, meta, v.([]interface{}))
-	if err != nil {
-		return nil, err
-	}
-	return item, nil
-}
+	// The API returns all monitored projects
+	monitoredProjects, _ := res["monitoredProjects"].([]interface{})
 
-func resourceMonitoringMonitoredProjectFindNestedObjectInList(d *schema.ResourceData, meta interface{}, items []interface{}) (index int, item map[string]interface{}, err error) {
-	expectedName, err := expandNestedMonitoringMonitoredProjectName(d.Get("name"), d, meta.(*transport_tpg.Config))
-	if err != nil {
-		return -1, nil, err
+	// Convert configured terraform monitored_project resource name to a ProjectNumber
+	expectedProject, configProjectErr := config.NewResourceManagerClient(config.UserAgent).Projects.Get(d.Get("name").(string)).Do()
+	if configProjectErr != nil {
+		return nil, configProjectErr
 	}
-	expectedFlattenedName := flattenNestedMonitoringMonitoredProjectName(expectedName, d, meta.(*transport_tpg.Config))
+	expectedProjectNumber := strconv.FormatInt(expectedProject.ProjectNumber, 10)
 
-	// Search list for this resource.
-	for idx, itemRaw := range items {
-		if itemRaw == nil {
-			continue
-		}
-		item := itemRaw.(map[string]interface{})
-
-		// Decode list item before comparing.
-		item, err := resourceMonitoringMonitoredProjectDecoder(d, meta, item)
-		if err != nil {
-			return -1, nil, err
-		}
+	log.Printf("[DEBUG] Scanning for ProjectNumber: %s.", expectedProjectNumber)
 
-		itemName := flattenNestedMonitoringMonitoredProjectName(item["name"], d, meta.(*transport_tpg.Config))
-		// IsEmptyValue check so that if one is nil and the other is "", that's considered a match
-		if !(tpgresource.IsEmptyValue(reflect.ValueOf(itemName)) && tpgresource.IsEmptyValue(reflect.ValueOf(expectedFlattenedName))) && !reflect.DeepEqual(itemName, expectedFlattenedName) {
-			log.Printf("[DEBUG] Skipping item with name= %#v, looking for %#v)", itemName, expectedFlattenedName)
+	// Iterate through the list of monitoredProjects to make sure one matches the configured monitored_project
+	for _, monitoredProjectRaw := range monitoredProjects {
+		if monitoredProjectRaw == nil {
 			continue
 		}
-		log.Printf("[DEBUG] Found item for resource %q: %#v)", d.Id(), item)
-		return idx, item, nil
-	}
-	return -1, nil, nil
-}
-func resourceMonitoringMonitoredProjectDecoder(d *schema.ResourceData, meta interface{}, res map[string]interface{}) (map[string]interface{}, error) {
-	config := meta.(*transport_tpg.Config)
-
-	expectedName, _ := expandNestedMonitoringMonitoredProjectName(d.Get("name"), d, config)
-	expectedFlattenedName := flattenNestedMonitoringMonitoredProjectName(expectedName, d, config)
-	_, isNumErr := tpgresource.StringToFixed64(expectedFlattenedName.(string))
-	expectProjectNumber := isNumErr == nil
-
-	name := res["name"].(string)
-	name = tpgresource.GetResourceNameFromSelfLink(name)
-
-	if expectProjectNumber {
-		res["name"] = name
-	} else if name != "" {
-		project, err := config.NewResourceManagerClient(config.UserAgent).Projects.Get(name).Do()
-		if err != nil {
-			return nil, err
+		monitoredProject := monitoredProjectRaw.(map[string]interface{})
+
+		// MonitoredProject names have the format locations/global/metricsScopes/[metricScopeProjectNumber]/projects/[projectNumber]
+		monitoredProjectName := monitoredProject["name"]
+
+		// `res` contains the MonitoredProjects of the relevant metrics scope
+		log.Printf("[DEBUG] Matching ProjectNumbers: %s to %s.", expectedProjectNumber, monitoredProjectName)
+		if strings.HasSuffix(monitoredProjectName.(string), fmt.Sprintf("/%s", expectedProjectNumber)) {
+			// Match found - set response object name to match
+			res["name"] = monitoredProjectName
+			log.Printf("[DEBUG] Matched ProjectNumbers: %s and %s.", expectedProjectNumber, monitoredProjectName)
+			return res, nil
 		}
-		res["name"] = project.ProjectId
 	}
-	return res, nil
+	log.Printf("[DEBUG] MonitoringMonitoredProject couldn't be matched.")
+	return nil, nil
 }
 
 func resourceMonitoringMonitoredProjectResourceV0() *schema.Resource {

google/services/monitoring/resource_monitoring_monitored_project_sweeper.go

@@ -108,7 +108,7 @@ func testSweepMonitoringMonitoredProject(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://monitoring.googleapis.com/v1/locations/global/metricsScopes/{{metrics_scope}}/projects/{{name}}"
+		deleteTemplate := "https://monitoring.googleapis.com/v1/{{name}}"
 		deleteUrl, err := tpgresource.ReplaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)