implement sweeper dependencies and parent relationships (GoogleCloudPlatform#13396)

Author: ScottSuarez

mmv1/api/resource.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/resource"
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/utils"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
 	"golang.org/x/exp/slices"
 )
@@ -1841,6 +1842,10 @@ func urlContainsOnlyAllowedKeys(templateURL string, allowedKeys []string) bool {
 }
 
 func (r Resource) ShouldGenerateSweepers() bool {
+	if !r.ExcludeSweeper && !utils.IsEmpty(r.Sweeper) {
+		return true
+	}
+
 	allowedKeys := []string{"project", "region", "location", "zone", "billing_account"}
 	if !urlContainsOnlyAllowedKeys(r.ListUrlTemplate(), allowedKeys) {
 		return false

mmv1/api/resource/sweeper.go

@@ -13,18 +13,59 @@
 
 package resource
 
-// Sweeper provides configuration for the test sweeper
+// Sweeper provides configuration for the test sweeper to clean up test resources
 type Sweeper struct {
-	// The field checked by sweeper to determine
-	// eligibility for deletion for generated resources
-	IdentifierField  string            `yaml:"identifier_field"`
-	Regions          []string          `yaml:"regions,omitempty"`
-	Prefixes         []string          `yaml:"prefixes,omitempty"`
-	URLSubstitutions []URLSubstitution `yaml:"url_substitutions,omitempty"`
+	// IdentifierField specifies which field in the resource object should be used
+	// to identify resources for deletion (typically "name" or "id")
+	IdentifierField string `yaml:"identifier_field"`
+
+	// Regions defines which regions to run the sweeper in
+	// If empty, defaults to just us-central1
+	Regions []string `yaml:"regions,omitempty"`
+
+	// Prefixes specifies name prefixes that identify resources eligible for sweeping
+	// Resources whose names start with any of these prefixes will be deleted
+	Prefixes []string `yaml:"prefixes,omitempty"`
+
+	// URLSubstitutions allows customizing URL parameters when listing resources
+	// Each map entry represents a set of key-value pairs to substitute in the URL template
+	URLSubstitutions []map[string]interface{} `yaml:"url_substitutions,omitempty"`
+
+	// Dependencies lists other resource types that must be swept before this one
+	Dependencies []string `yaml:"dependencies,omitempty"`
+
+	// Parent defines the parent-child relationship for hierarchical resources
+	// When specified, the sweeper will first collect parent resources before listing child resources
+	Parent *ParentResource `yaml:"parent,omitempty"`
 }
 
-// URLSubstitution represents a region-zone pair for URL substitution
-type URLSubstitution struct {
-	Region string `yaml:"region,omitempty"`
-	Zone   string `yaml:"zone,omitempty"`
+// ParentResource specifies how to handle parent-child resource dependencies
+type ParentResource struct {
+	// ResourceType is the parent resource type that will be used to find the parent sweeper
+	// Example: "GoogleContainerCluster"
+	ResourceType string `yaml:"resource_type"`
+
+	// ParentField specifies which field to extract from the parent resource
+	// Example: "name" or "id"
+	// Required unless Template is provided
+	ParentField string `yaml:"parent_field"`
+
+	// ParentFieldRegex is a regex pattern to apply to the parent field value
+	// The first capture group will be used as the final value
+	ParentFieldRegex string `yaml:"parent_field_regex"`
+
+	// ParentFieldExtractName when true indicates the parent field contains a self-link
+	// and only the resource name (portion after the last slash) should be used
+	ParentFieldExtractName bool `yaml:"parent_field_extract_name"`
+
+	// ChildField is the field in the child resource that needs to reference the parent
+	// Example: "cluster", "instance", etc.
+	ChildField string `yaml:"child_field"`
+
+	// Template provides a format string to construct the parent reference
+	// Variables in {{brackets}} will be replaced with values from the parent resource
+	// The special placeholder {{value}} is populated with the processed parent field value
+	// Example: "projects/{{project}}/locations/{{location}}/clusters/{{value}}"
+	// If specified, takes precedence over direct field mapping
+	Template string `yaml:"template"`
 }

mmv1/api/utils/utils.go

@@ -0,0 +1,71 @@
+package utils
+
+import "reflect"
+
+// IsEmpty checks if a value is meaningfully empty in a recursive way
+func IsEmpty(v interface{}) bool {
+	if v == nil {
+		return true
+	}
+
+	val := reflect.ValueOf(v)
+
+	// Handle pointers
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return true
+		}
+		return IsEmpty(val.Elem().Interface())
+	}
+
+	// Handle different types
+	switch val.Kind() {
+	case reflect.Struct:
+		// Check if all fields are empty
+		allEmpty := true
+		for i := 0; i < val.NumField(); i++ {
+			field := val.Field(i)
+			if field.CanInterface() && !IsEmpty(field.Interface()) {
+				allEmpty = false
+				break
+			}
+		}
+		return allEmpty
+
+	case reflect.Map:
+		if val.Len() == 0 {
+			return true
+		}
+		// Check if all map values are empty
+		allEmpty := true
+		iter := val.MapRange()
+		for iter.Next() {
+			if !IsEmpty(iter.Value().Interface()) {
+				allEmpty = false
+				break
+			}
+		}
+		return allEmpty
+
+	case reflect.Slice, reflect.Array:
+		if val.Len() == 0 {
+			return true
+		}
+		// Check if all elements are empty
+		allEmpty := true
+		for i := 0; i < val.Len(); i++ {
+			if !IsEmpty(val.Index(i).Interface()) {
+				allEmpty = false
+				break
+			}
+		}
+		return allEmpty
+
+	case reflect.Chan, reflect.Func, reflect.UnsafePointer:
+		return val.IsNil()
+
+	default:
+		// For simple types (int, string, etc.), check if it's a zero value
+		return val.IsZero()
+	}
+}

mmv1/api/utils/utils_test.go

@@ -0,0 +1,121 @@
+package utils
+
+import (
+	"testing"
+)
+
+type SimpleStruct struct {
+	Name  string
+	Value int
+}
+
+type NestedStruct struct {
+	Simple *SimpleStruct
+	Values []int
+	Map    map[string]interface{}
+}
+
+type ComplexStruct struct {
+	Nested    *NestedStruct
+	StructMap map[string]*SimpleStruct
+	StructArr []*SimpleStruct
+}
+
+func TestIsEmpty(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    interface{}
+		expected bool
+	}{
+		// Nil values
+		{"nil", nil, true},
+		{"nil pointer", (*SimpleStruct)(nil), true},
+
+		// Basic types
+		{"empty string", "", true},
+		{"non-empty string", "test", false},
+		{"zero int", 0, true},
+		{"non-zero int", 42, false},
+		{"false bool", false, true},
+		{"true bool", true, false},
+
+		// Slices and arrays
+		{"empty slice", []int{}, true},
+		{"nil slice", []int(nil), true},
+		{"non-empty slice", []int{1, 2, 3}, false},
+		{"slice with zero values", []int{0, 0, 0}, true},
+		{"slice with mixed values", []int{0, 1, 0}, false},
+
+		// Maps
+		{"empty map", map[string]int{}, true},
+		{"nil map", map[string]int(nil), true},
+		{"map with values", map[string]int{"one": 1, "two": 2}, false},
+		{"map with zero values", map[string]int{"one": 0, "two": 0}, true},
+		{"map with mixed values", map[string]int{"one": 0, "two": 2}, false},
+
+		// Simple struct
+		{"empty struct", SimpleStruct{}, true},
+		{"partially filled struct", SimpleStruct{Name: "test"}, false},
+		{"fully filled struct", SimpleStruct{Name: "test", Value: 42}, false},
+
+		// Pointers to simple structs
+		{"pointer to empty struct", &SimpleStruct{}, true},
+		{"pointer to partially filled struct", &SimpleStruct{Name: "test"}, false},
+
+		// Nested structs with nil fields
+		{"nested struct with all nil", NestedStruct{}, true},
+		{"nested struct with simple", NestedStruct{Simple: &SimpleStruct{Name: "test"}}, false},
+		{"nested struct with empty array", NestedStruct{Values: []int{}}, true},
+		{"nested struct with non-empty array", NestedStruct{Values: []int{1, 2}}, false},
+
+		// Complex nested scenarios
+		{
+			"complex all empty",
+			&ComplexStruct{
+				Nested: &NestedStruct{
+					Simple: &SimpleStruct{},
+					Values: []int{},
+					Map:    map[string]interface{}{},
+				},
+				StructMap: map[string]*SimpleStruct{},
+				StructArr: []*SimpleStruct{},
+			},
+			true,
+		},
+		{
+			"complex with one non-empty value",
+			&ComplexStruct{
+				Nested: &NestedStruct{
+					Simple: &SimpleStruct{Name: "test"},
+					Values: []int{},
+					Map:    map[string]interface{}{},
+				},
+				StructMap: map[string]*SimpleStruct{},
+				StructArr: []*SimpleStruct{},
+			},
+			false,
+		},
+		{
+			"complex with non-empty map",
+			&ComplexStruct{
+				Nested: &NestedStruct{
+					Simple: &SimpleStruct{},
+					Values: []int{},
+					Map:    map[string]interface{}{"key": "value"},
+				},
+				StructMap: map[string]*SimpleStruct{},
+				StructArr: []*SimpleStruct{},
+			},
+			false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result := IsEmpty(test.value)
+			if result != test.expected {
+				t.Errorf("IsEmpty(%v) = %v, expected %v", test.value, result, test.expected)
+			}
+		})
+	}
+}

mmv1/products/apigateway/ApiConfig.yaml

@@ -29,7 +29,6 @@ self_link: 'projects/{{project}}/locations/global/apis/{{api}}/configs/{{api_con
 create_url: 'projects/{{project}}/locations/global/apis/{{api}}/configs?apiConfigId={{api_config_id}}'
 update_verb: 'PATCH'
 update_mask: true
-
 read_query_params: '?view=FULL'
 timeouts:
   insert_minutes: 20
@@ -60,6 +59,12 @@ iam_policy:
 custom_code:
   extra_schema_entry: 'templates/terraform/extra_schema_entry/api_config.tmpl'
   encoder: 'templates/terraform/encoders/api_config.go.tmpl'
+sweeper:
+  parent:
+    resource_type: "google_api_gateway_api"
+    parent_field: "name"
+    parent_field_extract_name: true
+    child_field: "api"
 examples:
   - name: 'apigateway_api_config_basic'
     primary_resource_id: 'api_cfg'

mmv1/products/compute/ForwardingRule.yaml

@@ -48,6 +48,8 @@ custom_diff:
   - 'forwardingRuleCustomizeDiff'
 legacy_long_form_project: true
 sweeper:
+  dependencies:
+    - "google_network_connectivity_service_connection_policy"
   url_substitutions:
     - region: "us-west2"
     - region: "us-central1"

mmv1/products/compute/HealthCheck.yaml

@@ -54,6 +54,9 @@ custom_code:
   encoder: 'templates/terraform/encoders/health_check_type.tmpl'
 custom_diff:
   - 'healthCheckCustomizeDiff'
+sweeper:
+  dependencies:
+    - "google_compute_subnetwork"
 examples:
   - name: 'health_check_tcp'
     primary_resource_id: 'tcp-health-check'

mmv1/products/compute/InstantSnapshot.yaml

@@ -49,6 +49,9 @@ iam_policy:
   allowed_iam_role: 'roles/compute.storageAdmin'
   import_format:
     - 'projects/{{project}}/zones/{{zone}}/instantSnapshots/{{name}}'
+sweeper:
+  url_substitutions:
+    - zone: "us-central1-a"
 examples:
   - name: 'instant_snapshot_basic'
     primary_resource_id: 'default'

mmv1/products/compute/Subnetwork.yaml

@@ -71,6 +71,8 @@ custom_diff:
   - 'customdiff.ForceNewIfChange("ip_cidr_range", IsShrinkageIpCidr)'
   - 'sendSecondaryIpRangeIfEmptyDiff'
 sweeper:
+  dependencies:
+    - "google_compute_forwarding_rule"
   url_substitutions:
     - region: "us-west2"
     - region: "us-central1"

mmv1/products/discoveryengine/ChatEngine.yaml

@@ -44,6 +44,12 @@ async:
     resource_inside_response: true
 custom_code:
   encoder: 'templates/terraform/encoders/discovery_engine_chat_engine_hardcode_solution_type.go.tmpl'
+sweeper:
+  url_substitutions:
+    - collection_id: default_collection
+      region: global
+    - collection_id: default_collection
+      region: eu
 examples:
   - name: 'discoveryengine_chat_engine_basic'
     primary_resource_id: 'primary'

mmv1/products/discoveryengine/DataStore.yaml

@@ -44,6 +44,8 @@ async:
     resource_inside_response: false
 custom_code:
 sweeper:
+  dependencies:
+    - 'google_discovery_engine_chat_engine'
   url_substitutions:
     - region: "eu"
     - region: "global"

mmv1/products/networkconnectivity/ServiceConnectionPolicy.yaml

@@ -46,6 +46,8 @@ async:
 custom_code:
   update_encoder: 'templates/terraform/encoders/service_connection_policy.go.tmpl'
 sweeper:
+  prefixes:
+    - "gcp-memorystore"
   url_substitutions:
     - region: "us-central1"
     - region: "us-east1"

mmv1/products/networkservices/GrpcRoute.yaml

@@ -45,6 +45,9 @@ async:
 custom_code:
 schema_version: 1
 state_upgraders: true
+sweeper:
+  url_substitutions:
+    - region: "global"
 examples:
   - name: 'network_services_grpc_route_basic'
     primary_resource_id: 'default'

mmv1/products/networkservices/Mesh.yaml

@@ -47,6 +47,9 @@ async:
 custom_code:
 schema_version: 1
 state_upgraders: true
+sweeper:
+  url_substitutions:
+    - region: "global"
 examples:
   - name: 'network_services_mesh_basic'
     primary_resource_id: 'default'