Merge branch 'GoogleCloudPlatform:main' into colab-schedule

Author: bcreddy-gcp

.github/workflows/build-downstream.yml

@@ -32,7 +32,7 @@ jobs:
 
       # Cache Go modules
       - name: Cache Go modules
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/downstreams.yml

@@ -8,7 +8,7 @@ on:
       - 'FEATURE-BRANCH-*'
   merge_group:
     types: [checks_requested]
-
+  pull_request:
 
 concurrency:
   group: ${{ github.event_name == 'merge_group' && format('merge-group-{0}', github.event.merge_group.head_sha) || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('commit-{0}', github.sha) }}

docs/content/code-review/create-pr.md

@@ -52,7 +52,7 @@ VCR test failures that do not immediately seem related to your PR are most likel
    git checkout modular-magician/auto-pr-PR_NUMBER
    make test
    make lint
-   make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool'
+   make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool_basic$$'
    ```
    Replace PR_NUMBER with your PR's ID.
    {{< /tab >}}
@@ -65,7 +65,7 @@ VCR test failures that do not immediately seem related to your PR are most likel
    git checkout modular-magician/auto-pr-PR_NUMBER
    make test
    make lint
-   make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool'
+   make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool_basic$$'
    ```
    Replace PR_NUMBER with your PR's ID.
    {{< /tab >}}

docs/content/test/run-tests.md

@@ -59,16 +59,22 @@ aliases:
 
 1. Run acceptance tests for only modified resources. (Full test runs can take over 9 hours.) See [Go's documentation](https://pkg.go.dev/cmd/go#hdr-Testing_flags) for more information about `-run` and other flags.
 
+    ```bash
+    make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool_basic$$'
+    ```
+
+    To run all tests matching, e.g., `TestAccContainerNodePool*`, omit the trailing `$$`:
+
     ```bash
     make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool'
     ```
 
-> **Note:** Acceptance tests create actual infrastructure which can incur costs. Acceptance tests may not clean up after themselves if interrupted, so you may want to check for stray resources and / or billing charges.
+    > **Note:** Acceptance tests create actual infrastructure which can incur costs. Acceptance tests may not clean up after themselves if interrupted, so you may want to check for stray resources and / or billing charges.
 
 1. Optional: Save verbose test output (including API requests and responses) to a file for analysis.
 
     ```bash
-    TF_LOG=DEBUG make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool_basic' > output.log
+    TF_LOG=DEBUG make testacc TEST=./google/services/container TESTARGS='-run=TestAccContainerNodePool_basic$$' > output.log
     ```
 
 1. Optional: Debug tests with [Delve](https://github.com/go-delve/delve). See [`dlv test` documentation](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_test.md) for information about available flags.
@@ -95,12 +101,19 @@ aliases:
     ```bash
     make testacc TEST=./google-beta/services/container TESTARGS='-run=TestAccContainerNodePool'
     ```
-> **Note:** Acceptance tests create actual infrastructure which can incur costs. Acceptance tests may not clean up after themselves if interrupted, so you may want to check for stray resources and / or billing charges.
+
+    To run all tests matching, e.g., `TestAccContainerNodePool*`, omit the trailing `$$`:
+
+    ```bash
+    make testacc TEST=./google-beta/services/container TESTARGS='-run=TestAccContainerNodePool'
+    ```
+
+    > **Note:** Acceptance tests create actual infrastructure which can incur costs. Acceptance tests may not clean up after themselves if interrupted, so you may want to check for stray resources and / or billing charges.
 
 1. Optional: Save verbose test output to a file for analysis.
 
     ```bash
-    TF_LOG=DEBUG make testacc TEST=./google-beta/services/container TESTARGS='-run=TestAccContainerNodePool_basic' > output.log
+    TF_LOG=DEBUG make testacc TEST=./google-beta/services/container TESTARGS='-run=TestAccContainerNodePool_basic$$' > output.log
     ```
 
 1. Optional: Debug tests with [Delve](https://github.com/go-delve/delve). See [`dlv test` documentation](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_test.md) for information about available flags.
@@ -290,6 +303,32 @@ Configure Terraform to use locally-built binaries for `google` and `google-beta`
     TF_LOG=DEBUG TF_LOG_PATH=output.log TF_CLI_CONFIG_FILE="$HOME/tf-dev-override.tfrc" terraform apply
     ```
 
+### Run Tests with VCR Locally
+
+VCR tests record HTTP request/response interactions in cassettes and replay them in future runs without calling the real API.
+
+Running tests in `REPLAYING` mode locally can sometimes be useful. In particular, it can allow you to test more quickly, cheaply, and without spinning up real infrastructure, once you've got an initial recording.
+
+It can also be helpful for debugging tests that seem to work locally, but fail in CI in replaying mode.
+   
+VCR is controlled via two variables:
+- `VCR_MODE`: `REPLAYING` or `RECORDING` mode 
+- `VCR_PATH`: Path where recorded cassettes are stored. 
+
+Ensure both variables are configured to properly trigger VCR tests locally.
+
+If you don't already have an existing cassette that's up to date, first do a run in `RECORDING` mode:
+   
+```bash
+VCR_PATH=$HOME/.vcr/ VCR_MODE=RECORDING  make testacc TEST=./google/services/alloydb TESTARGS='-run=TestAccContainerNodePool_basic$$'
+```
+
+Now run the same test again in `REPLAYING` mode:
+
+```bash
+VCR_PATH=$HOME/.vcr/ VCR_MODE=REPLAYING make testacc TEST=./google/services/alloydb TESTARGS='-run=TestAccContainerNodePool_basic$$'
+```
+
 ### Cleanup
 
 To stop using developer overrides, stop setting `TF_CLI_CONFIG_FILE` in the commands you are executing.

mmv1/api/resource.go

@@ -337,7 +337,8 @@ type Resource struct {
 	// fine-grained resources and legacy resources.
 	ApiResourceTypeKind string `yaml:"api_resource_type_kind,omitempty"`
 
-	ImportPath string `yaml:"-"`
+	ImportPath     string `yaml:"-"`
+	SourceYamlFile string `yaml:"-"`
 }
 
 func (r *Resource) UnmarshalYAML(unmarshal func(any) error) error {
@@ -1787,3 +1788,45 @@ func (r Resource) CaiIamAssetNameTemplate(productBackendName string) string {
 	}
 	return fmt.Sprintf("//%s.googleapis.com/%s/{{%s}}", productBackendName, caiBaseUrl, r.IamParentResourceName())
 }
+
+func urlContainsOnlyAllowedKeys(templateURL string, allowedKeys []string) bool {
+	// Create regex to match anything between {{ and }}
+	re := regexp.MustCompile(`{{\s*([^}]+)\s*}}`)
+
+	// Find all matches in the template URL
+	matches := re.FindAllStringSubmatch(templateURL, -1)
+
+	// Create a map of allowed keys for O(1) lookup
+	allowedKeysMap := make(map[string]bool)
+	for _, key := range allowedKeys {
+		allowedKeysMap[key] = true
+	}
+
+	// Check each found key against the allowed keys
+	for _, match := range matches {
+		if len(match) < 2 {
+			continue
+		}
+
+		// Trim spaces from the key
+		key := strings.TrimSpace(match[1])
+
+		// If the key isn't in our allowed list, return false
+		if !allowedKeysMap[key] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (r Resource) ShouldGenerateSweepers() bool {
+	allowedKeys := []string{"project", "region", "location", "zone", "billing_account"}
+	if !urlContainsOnlyAllowedKeys(r.ListUrlTemplate(), allowedKeys) {
+		return false
+	}
+	if r.ExcludeSweeper || r.CustomCode.CustomDelete != "" || r.CustomCode.PreDelete != "" || r.CustomCode.PostDelete != "" || r.ExcludeDelete {
+		return false
+	}
+	return true
+}

mmv1/main.go

@@ -256,6 +256,7 @@ func GenerateProduct(productChannel chan string, providerToGenerate provider.Pro
 
 		resource := &api.Resource{}
 		api.Compile(resourceYamlPath, resource, overrideDirectory)
+		resource.SourceYamlFile = resourceYamlPath
 
 		resource.TargetVersionName = *version
 		resource.Properties = resource.AddLabelsRelatedFields(resource.PropertiesWithExcluded(), nil)

mmv1/products/accesscontextmanager/ServicePerimeter.yaml

@@ -384,6 +384,15 @@ properties:
                       - name: 'accessLevel'
                         type: String
                         description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+                      - name: 'resource'
+                        type: String
+                        description: |
+                          A Google Cloud resource that is allowed to egress the perimeter.
+                          Requests from these resources are allowed to access data outside the perimeter.
+                          Currently only projects are allowed. Project format: `projects/{project_number}`.
+                          The resource may be in any Google Cloud organization, not just the
+                          organization that the perimeter is defined in. `*` is not allowed, the
+                          case of allowing all Google Cloud resources only is not supported.
                 - name: 'sourceRestriction'
                   type: Enum
                   description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'
@@ -693,6 +702,15 @@ properties:
                       - name: 'accessLevel'
                         type: String
                         description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+                      - name: 'resource'
+                        type: String
+                        description: |
+                          A Google Cloud resource that is allowed to egress the perimeter.
+                          Requests from these resources are allowed to access data outside the perimeter.
+                          Currently only projects are allowed. Project format: `projects/{project_number}`.
+                          The resource may be in any Google Cloud organization, not just the
+                          organization that the perimeter is defined in. `*` is not allowed, the
+                          case of allowing all Google Cloud resources only is not supported.
                 - name: 'sourceRestriction'
                   type: Enum
                   description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'

mmv1/products/accesscontextmanager/ServicePerimeterDryRunEgressPolicy.yaml

@@ -128,6 +128,15 @@ properties:
             - name: 'accessLevel'
               type: String
               description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+            - name: 'resource'
+              type: String
+              description: |
+                A Google Cloud resource that is allowed to egress the perimeter.
+                Requests from these resources are allowed to access data outside the perimeter.
+                Currently only projects are allowed. Project format: `projects/{project_number}`.
+                The resource may be in any Google Cloud organization, not just the
+                organization that the perimeter is defined in. `*` is not allowed, the
+                case of allowing all Google Cloud resources only is not supported.
       - name: 'sourceRestriction'
         type: Enum
         description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'

mmv1/products/accesscontextmanager/ServicePerimeterEgressPolicy.yaml

@@ -125,6 +125,15 @@ properties:
             - name: 'accessLevel'
               type: String
               description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+            - name: 'resource'
+              type: String
+              description: |
+                A Google Cloud resource that is allowed to egress the perimeter.
+                Requests from these resources are allowed to access data outside the perimeter.
+                Currently only projects are allowed. Project format: `projects/{project_number}`.
+                The resource may be in any Google Cloud organization, not just the
+                organization that the perimeter is defined in. `*` is not allowed, the
+                case of allowing all Google Cloud resources only is not supported.
       - name: 'sourceRestriction'
         type: Enum
         description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'

mmv1/products/accesscontextmanager/ServicePerimeters.yaml

@@ -373,6 +373,15 @@ properties:
                             - name: 'accessLevel'
                               type: String
                               description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+                            - name: 'resource'
+                              type: String
+                              description: |
+                                A Google Cloud resource that is allowed to egress the perimeter.
+                                Requests from these resources are allowed to access data outside the perimeter.
+                                Currently only projects are allowed. Project format: `projects/{project_number}`.
+                                The resource may be in any Google Cloud organization, not just the
+                                organization that the perimeter is defined in. `*` is not allowed, the
+                                case of allowing all Google Cloud resources only is not supported.
                       - name: 'sourceRestriction'
                         type: Enum
                         description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'
@@ -674,6 +683,15 @@ properties:
                             - name: 'accessLevel'
                               type: String
                               description: 'An AccessLevel resource name that allows resources outside the ServicePerimeter to be accessed from the inside.'
+                            - name: 'resource'
+                              type: String
+                              description: |
+                                A Google Cloud resource that is allowed to egress the perimeter.
+                                Requests from these resources are allowed to access data outside the perimeter.
+                                Currently only projects are allowed. Project format: `projects/{project_number}`.
+                                The resource may be in any Google Cloud organization, not just the
+                                organization that the perimeter is defined in. `*` is not allowed, the
+                                case of allowing all Google Cloud resources only is not supported.
                       - name: 'sourceRestriction'
                         type: Enum
                         description: 'Whether to enforce traffic restrictions based on `sources` field. If the `sources` field is non-empty, then this field must be set to `SOURCE_RESTRICTION_ENABLED`.'

mmv1/products/apphub/Application.yaml

@@ -40,6 +40,11 @@ custom_code:
   constants: 'templates/terraform/constants/apphub_application.go.tmpl'
 custom_diff:
   - 'apphubApplicationCustomizeDiff'
+sweeper:
+  regions:
+    - "us-central1"
+    - "us-east1"
+    - "global"
 examples:
   - name: 'apphub_application_basic'
     primary_resource_id: 'example'

mmv1/products/artifactregistry/Repository.yaml

@@ -54,6 +54,10 @@ custom_code:
   constants: 'templates/terraform/constants/artifact_registry_repository.go.tmpl'
   encoder: 'templates/terraform/encoders/location_from_region.go.tmpl'
   pre_create: 'templates/terraform/pre_create/artifact_registry_remote_repository.go.tmpl'
+sweeper:
+  regions:
+    - "us-central1"
+    - "us"
 examples:
   - name: 'artifact_registry_repository_basic'
     primary_resource_id: 'my-repo'