Merge branch 'GoogleCloudPlatform:main' into mc-parameterized

nehalk-tf · web-flow · commit 04445c313320 · 2024-10-24T11:30:03.000-07:00
diff --git a/mmv1/products/apphub/Application.yaml b/mmv1/products/apphub/Application.yaml
@@ -43,14 +43,30 @@ async:
     path: 'error'
     message: 'message'
 custom_code:
+  constants: 'templates/terraform/constants/apphub_application.go.tmpl'
+custom_diff:
+  - 'apphubApplicationCustomizeDiff'
 examples:
-  - name: 'application_basic'
+  - name: 'apphub_application_basic'
+    primary_resource_id: 'example'
+    vars:
+      application_id: 'example-application'
+      location: 'us-east1'
+      scope_type: 'REGIONAL'
+    test_vars_overrides:
+      'location': '"us-east1"'
+      'scope_type': '"REGIONAL"'
+  - name: 'apphub_application_global_basic'
     config_path: 'templates/terraform/examples/apphub_application_basic.tf.tmpl'
     primary_resource_id: 'example'
     vars:
       application_id: 'example-application'
-  - name: 'application_full'
-    config_path: 'templates/terraform/examples/apphub_application_full.tf.tmpl'
+      location: 'global'
+      scope_type: 'GLOBAL'
+    test_vars_overrides:
+      'location': '"global"'
+      'scope_type': '"GLOBAL"'
+  - name: 'apphub_application_full'
     primary_resource_id: 'example2'
     vars:
       application_id: 'example-application'
@@ -171,10 +187,11 @@ properties:
     properties:
       - name: 'type'
         type: Enum
-        description: "Required. Scope Type. \n Possible values:\nREGIONAL"
+        description: "Required. Scope Type. \n Possible values:\nREGIONAL\nGLOBAL"
         required: true
         enum_values:
           - 'REGIONAL'
+          - 'GLOBAL'
   - name: 'uid'
     type: String
     description: 'Output only. A universally unique identifier (in UUID4 format) for
diff --git a/mmv1/products/compute/ServiceAttachment.yaml b/mmv1/products/compute/ServiceAttachment.yaml
@@ -169,7 +169,6 @@ properties:
           type: Integer
           description: |
             The number of consumer Network Connectivity Center spokes that the connected Private Service Connect endpoint has propagated to.
-          min_version: 'beta'
           output: true
   - name: 'targetService'
     type: String
@@ -266,5 +265,4 @@ properties:
       If the connection preference of the service attachment is ACCEPT_AUTOMATIC, the limit applies to each project that contains a connected endpoint.
 
       If unspecified, the default propagated connection limit is 250.
-    min_version: 'beta'
     default_from_api: true
diff --git a/mmv1/products/gkebackup/RestorePlan.yaml b/mmv1/products/gkebackup/RestorePlan.yaml
@@ -59,9 +59,9 @@ examples:
       name: 'restore-all-ns'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -74,9 +74,9 @@ examples:
       name: 'rollback-ns'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -89,9 +89,9 @@ examples:
       name: 'rollback-app'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -104,9 +104,9 @@ examples:
       name: 'all-groupkinds'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -119,9 +119,9 @@ examples:
       name: 'rename-ns'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -134,9 +134,9 @@ examples:
       name: 'transform-rule'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -149,9 +149,9 @@ examples:
       name: 'gitops-mode'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -164,9 +164,9 @@ examples:
       name: 'restore-order'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
@@ -179,9 +179,9 @@ examples:
       name: 'volume-res'
       network_name: 'default'
       subnetwork_name: 'default'
+      deletion_protection: 'true'
     test_env_vars:
       project: 'PROJECT_NAME'
-      deletion_protection: 'true'
     test_vars_overrides:
       'deletion_protection': 'false'
       'network_name': 'acctest.BootstrapSharedTestNetwork(t, "gke-cluster")'
diff --git a/mmv1/products/spanner/Instance.yaml b/mmv1/products/spanner/Instance.yaml
@@ -228,6 +228,40 @@ properties:
               should be trying to achieve for the instance.
               This number is on a scale from 0 (no utilization) to 100 (full utilization).
 
+      - name: 'asymmetricAutoscalingOptions'
+        type: Array
+        description: |
+          Asymmetric autoscaling options for specific replicas.
+        item_type:
+          type: NestedObject
+          properties:
+            - name: 'replicaSelection'
+              type: NestedObject
+              required: true
+              properties:
+                - name: 'location'
+                  type: String
+                  required: true
+                  description: |
+                    The location of the replica to apply asymmetric autoscaling options.
+            - name: 'overrides'
+              type: NestedObject
+              required: true
+              properties:
+                - name: 'autoscalingLimits'
+                  type: NestedObject
+                  required: true
+                  properties:
+                    - name: 'minNodes'
+                      type: Integer
+                      required: true
+                      description: |
+                        The minimum number of nodes for this specific replica.
+                    - name: 'maxNodes'
+                      type: Integer
+                      required: true
+                      description: |
+                        The maximum number of nodes for this specific replica.
   - name: 'edition'
     type: Enum
     description: |
diff --git a/mmv1/templates/terraform/constants/apphub_application.go.tmpl b/mmv1/templates/terraform/constants/apphub_application.go.tmpl
@@ -0,0 +1,17 @@
+func apphubApplicationCustomizeDiff(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error {
+    if diff.HasChange("location") || diff.HasChange("scope.0.type") {
+        location := diff.Get("location")
+        scope_type := diff.Get("scope.0.type")
+
+        if scope_type == "GLOBAL" {
+            if location != "global" {
+                return fmt.Errorf("Error validating location %s with %s scope type", location, scope_type)
+            }
+        } else {
+            if location == "global" {
+                return fmt.Errorf("Error validating location %s with %s scope type", location, scope_type)
+            }
+        }
+    }
+    return nil
+}
diff --git a/mmv1/templates/terraform/encoders/spanner_instance_update.go.tmpl b/mmv1/templates/terraform/encoders/spanner_instance_update.go.tmpl
@@ -46,6 +46,9 @@ if d.HasChange("autoscaling_config") {
 		if d.HasChange("autoscaling_config.0.autoscaling_targets.0.storage_utilization_percent") {
 			updateMask = append(updateMask, "autoscalingConfig.autoscalingTargets.storageUtilizationPercent")
 		}
+		if d.HasChange("autoscaling_config.0.asymmetric_autoscaling_options") {
+			updateMask = append(updateMask, "autoscalingConfig.asymmetricAutoscalingOptions")
+		}
 	}
 }
 newObj["fieldMask"] = strings.Join(updateMask, ",")
diff --git a/mmv1/templates/terraform/examples/apphub_application_basic.tf.tmpl b/mmv1/templates/terraform/examples/apphub_application_basic.tf.tmpl
@@ -1,7 +1,7 @@
 resource "google_apphub_application" "{{$.PrimaryResourceId}}" {
-  location = "us-east1"
+  location = "{{index $.Vars "location"}}"
   application_id = "{{index $.Vars "application_id"}}"
   scope {
-    type = "REGIONAL"
+    type = "{{index $.Vars "scope_type"}}"
   }
 }
diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt
@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import generated.ServiceParallelism
@@ -74,6 +75,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
                 downloadTerraformBinary()
                 runAcceptanceTests()
                 saveArtifactsToGCS()
+                archiveArtifactsIfOverLimit() // Must be after push to GCS step, as this step impacts debug log files
             }
 
             features {
@@ -98,7 +100,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true
diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt
@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import jetbrains.buildServer.configs.kotlin.BuildType
@@ -102,7 +103,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true
diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt
@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import jetbrains.buildServer.configs.kotlin.BuildType
@@ -77,7 +78,7 @@ class VcrDetails(private val providerName: String, private val buildId: String,
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true
diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt
@@ -137,7 +137,7 @@ fun BuildSteps.saveArtifactsToGCS() {
         name = "Tasks after running nightly tests: push artifacts(debug logs) to GCS"
         scriptContent = """
             #!/bin/bash
-            echo "Post-test step - storge artifacts(debug logs) to GCS"
+            echo "Post-test step - storage artifacts(debug logs) to GCS"
 
             # Authenticate gcloud CLI
             echo "${'$'}{GOOGLE_CREDENTIALS_GCS}" > google-account.json
@@ -173,4 +173,56 @@ fun BuildSteps.saveArtifactsToGCS() {
         // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see:
         // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates
     })
-}
+}
+
+// The S3 plugin we use to upload artifacts to S3 (enabling them to be accessed via the TeamCity UI later) has a limit of
+// 1000 artifacts to be uploaded at a time. To avoid a situation where no artifacts are uploaded as a result of exceeding this
+// limit we archive all the debug logs if they equal or exceed 1000 for a given build.
+fun BuildSteps.archiveArtifactsIfOverLimit() {
+    step(ScriptBuildStep {
+        name = "Tasks after running nightly tests: archive artifacts(debug logs) if there are >=1000 before S3 upload"
+        scriptContent = """
+            #!/bin/bash
+            echo "Post-test step - archive artifacts(debug logs) if there are >=1000 before S3 upload"
+
+            # Get number of artifacts created
+            ARTIFACT_COUNT=$(ls %teamcity.build.checkoutDir%/debug* | wc -l | grep -o -E '[0-9]+')
+
+            if test ${'$'}ARTIFACT_COUNT -lt "1000"; then
+                echo "Found <1000 debug artifacts; we won't archive them before upload to S3"
+                exit 0
+            fi
+
+            echo "Found >=1000 debug artifacts; archiving before upload to S3"
+
+            # Make tarball of all debug logs
+            # Name should look similar to: debug-google-d2503f7-253644-TerraformProviders_GoogleCloud_GOOGLE_NIGHTLYTESTS_GOOGLE_PACKAGE_ACCESSAPPROVAL.tar.gz
+            cd %teamcity.build.checkoutDir%
+            ARCHIVE_NAME=debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%system.teamcity.buildType.id%-archive.tar.gz
+            tar -cf ${'$'}ARCHIVE_NAME ./debug*
+
+            # Fail loudly if archive not made as expected
+            if [ ! -f ${'$'}ARCHIVE_NAME ]; then
+                echo "Archive file ${'$'}ARCHIVE_NAME not found!"
+
+                # Allow sanity checking
+                echo "Listing contents of %teamcity.build.checkoutDir%"
+                ls
+
+                exit 1
+            fi
+
+            # Remove all debug logs. These are all .txt files due to the effects of TF_LOG_PATH_MASK.
+            rm ./debug*.txt
+
+            # Allow sanity checking
+            echo "Listing files matching the artifact rule value %teamcity.build.checkoutDir%/debug*"
+            ls debug*
+
+            echo "Finished"
+        """.trimIndent()
+        // ${'$'} is required to allow creating a script in TeamCity that contains
+        // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see:
+        // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates
+    })
+}
diff --git a/mmv1/third_party/terraform/.teamcity/components/constants.kt b/mmv1/third_party/terraform/.teamcity/components/constants.kt
@@ -43,3 +43,8 @@ const val ProjectSweeperName = "Project Sweeper"
 const val NightlyTestsProjectId = "NightlyTests"
 const val MMUpstreamProjectId = "MMUpstreamTests"
 const val VcrRecordingProjectId = "VCRRecording"
+
+// Artifact rules controls which artifacts are uploaded to S3
+// https://www.jetbrains.com/help/teamcity/2024.07/configuring-general-settings.html#Artifact+Paths
+// The value below lacks a file extension, to allow upload of individual .txt files or a single .tar.gz file
+const val ArtifactRules = "%teamcity.build.checkoutDir%/debug*"
diff --git a/mmv1/third_party/terraform/services/apphub/resource_apphub_application_test.go b/mmv1/third_party/terraform/services/apphub/resource_apphub_application_test.go
diff --git a/mmv1/third_party/terraform/services/compute/resource_compute_service_attachment_test.go.tmpl b/mmv1/third_party/terraform/services/compute/resource_compute_service_attachment_test.go.tmpl
diff --git a/mmv1/third_party/terraform/services/spanner/resource_spanner_instance_test.go b/mmv1/third_party/terraform/services/spanner/resource_spanner_instance_test.go

Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,9 @@ if d.HasChange("autoscaling_config") {`
`46`	`46`	`if d.HasChange("autoscaling_config.0.autoscaling_targets.0.storage_utilization_percent") {`
`47`	`47`	`updateMask = append(updateMask, "autoscalingConfig.autoscalingTargets.storageUtilizationPercent")`
`48`	`48`	`}`
	`49`	`+ if d.HasChange("autoscaling_config.0.asymmetric_autoscaling_options") {`
	`50`	`+ updateMask = append(updateMask, "autoscalingConfig.asymmetricAutoscalingOptions")`
	`51`	`+ }`
`49`	`52`	`}`
`50`	`53`	`}`
`51`	`54`	`newObj["fieldMask"] = strings.Join(updateMask, ",")`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`resource "google_apphub_application" "{{$.PrimaryResourceId}}" {`
`2`		`- location = "us-east1"`
	`2`	`+ location = "{{index $.Vars "location"}}"`
`3`	`3`	`application_id = "{{index $.Vars "application_id"}}"`
`4`	`4`	`scope {`
`5`		`- type = "REGIONAL"`
	`5`	`+ type = "{{index $.Vars "scope_type"}}"`
`6`	`6`	`}`
`7`	`7`	`}`