Merge branch 'GoogleCloudPlatform:main' into main

Author: ashwinsshetty

.ci/containers/README.md

@@ -24,6 +24,6 @@ Before you begin, set up Docker (including configuring it to [authenticate with
    ```
 4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
 5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
-   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.erb` will trigger builds for TPG, TPGB, and TGC.
+   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.tmpl` will trigger builds for TPG, TPGB, and TGC.
 6. Create a PR with these changes.
 7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)

.ci/gcb-vcr-nightly.yml

@@ -2,10 +2,15 @@
 steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-vcr-nightly
-      entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
-      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GITHUB_TOKEN_CLASSIC"]
+      env:
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+        - "USER=magician"
       args:
-          - $BUILD_ID
+        - 'vcr-cassette-update'
+        - $BUILD_ID
 
 # Long timeout to enable waiting on VCR test
 timeout: 20000s
@@ -39,3 +44,5 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
+    - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
+      env: GITHUB_TOKEN_CLASSIC

.ci/magician/cmd/templates/vcr/vcr_cassettes_update_recording.tmpl

@@ -5,6 +5,7 @@ The provider crashed while running the VCR tests in RECORDING mode
 {{- else -}}
 #################################
 RECORDING Tests Report
+#################################
 
 {{if .RecordingResult.PassedTests}}
 Tests passed during RECORDING mode:
@@ -18,7 +19,7 @@ Tests failed during RECORDING mode:
 
 {{if .HasTerminatedTests}}Several tests got terminated during RECORDING mode{{end}}
 
-{{if .RecordingErr}}Errors occurred during RECORDING mode.{{end}}
+{{if .RecordingErr}}Errors occurred during RECORDING mode: {{.RecordingErr}}.{{end}}
 
 {{if .AllRecordingPassed}}All tests passed!{{end}}
 

.ci/magician/cmd/templates/vcr/vcr_cassettes_update_replaying.tmpl

@@ -3,6 +3,8 @@ The provider crashed while running the VCR tests in REPLAYING mode
 {{- else}}
 #################################
 Tests Analytics
+#################################
+
 Total tests: {{add (add (len .ReplayingResult.PassedTests) (len .ReplayingResult.SkippedTests)) (len .ReplayingResult.FailedTests) }}
 Passed tests: {{len .ReplayingResult.PassedTests}}
 Skipped tests: {{len .ReplayingResult.SkippedTests}}
@@ -15,7 +17,7 @@ Affected tests list:
 {{- end}}
 {{if .ReplayingErr}}
 #################################
-Errors occurred during REPLAYING mode.
+Errors occurred during REPLAYING mode: {{.ReplayingErr}}.
 #################################
 {{- end}}
 {{if .AllReplayingPassed}}

.ci/magician/cmd/vcr_cassette_update.go

@@ -133,12 +133,12 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
 	// upload replay build and test logs
 	buildLogPath := filepath.Join(rnr.GetCWD(), "testlogs", fmt.Sprintf("%s_test.log", vcr.Replaying.Lower()))
 	if _, err := uploadLogsToGCS(buildLogPath, bucketPrefix+"/logs/replaying/", rnr); err != nil {
-		return fmt.Errorf("error uploading replaying test log: %w", err)
+		fmt.Printf("Warning: error uploading replaying test log: %s\n", err)
 	}
 
 	testLogPath := vt.LogPath(vcr.Replaying, provider.Beta)
 	if _, err := uploadLogsToGCS(filepath.Join(testLogPath, "*"), bucketPrefix+"/logs/build-log/", rnr); err != nil {
-		return fmt.Errorf("error uploading replaying build log: %w", err)
+		fmt.Printf("Warning: error uploading replaying build log: %s\n", err)
 	}
 
 	replayingData := vcrCassetteUpdateReplayingResult{
@@ -169,17 +169,19 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
 		// uploading cassettes failed because recording not work
 		buildLogPath := filepath.Join(rnr.GetCWD(), "testlogs", fmt.Sprintf("%s_test.log", vcr.Recording.Lower()))
 		if _, err := uploadLogsToGCS(buildLogPath, bucketPrefix+"/logs/recording/", rnr); err != nil {
-			return fmt.Errorf("error uploading recording test log: %w", err)
+			fmt.Printf("Warning: error uploading recording test log: %s\n", err)
 		}
 
 		testLogPath := vt.LogPath(vcr.Recording, provider.Beta)
 		if _, err := uploadLogsToGCS(filepath.Join(testLogPath, "*"), bucketPrefix+"/logs/build-log/", rnr); err != nil {
-			return fmt.Errorf("error uploading recording build log: %w", err)
+			fmt.Printf("Warning: error uploading recording build log: %s\n", err)
 		}
+
 		if len(recordingResult.PassedTests) > 0 {
 			cassettesPath := vt.CassettePath(provider.Beta)
-			if _, err := uploadCassettesToGCS(cassettesPath, "gs://ci-vcr-cassettes/beta/fixtures/", rnr); err != nil {
-				return fmt.Errorf("error uploading cassettes: %w", err)
+			if _, err := uploadCassettesToGCS(cassettesPath+"/*", "gs://ci-vcr-cassettes/beta/fixtures/", rnr); err != nil {
+				// There could be cases that the tests do not generate any cassettes.
+				fmt.Printf("Warning: error uploading cassettes: %s\n", err)
 			}
 		} else {
 			fmt.Println("No tests passed in recording mode, not uploading cassettes.")

.ci/magician/cmd/vcr_cassette_update_test.go

@@ -34,6 +34,8 @@ func TestFormatVCRCassettesUpdateReplaying(t *testing.T) {
 				[]string{
 					"#################################",
 					"Tests Analytics",
+					"#################################",
+					"",
 					"Total tests: 5",
 					"Passed tests: 2",
 					"Skipped tests: 1",
@@ -47,7 +49,8 @@ func TestFormatVCRCassettesUpdateReplaying(t *testing.T) {
 					"#################################",
 					"",
 					"#################################",
-					"Errors occurred during REPLAYING mode.", "#################################",
+					"Errors occurred during REPLAYING mode: some error.",
+					"#################################",
 				},
 				"\n",
 			),
@@ -65,6 +68,8 @@ func TestFormatVCRCassettesUpdateReplaying(t *testing.T) {
 				[]string{
 					"#################################",
 					"Tests Analytics",
+					"#################################",
+					"",
 					"Total tests: 3",
 					"Passed tests: 2",
 					"Skipped tests: 1",
@@ -74,7 +79,8 @@ func TestFormatVCRCassettesUpdateReplaying(t *testing.T) {
 					"",
 					"",
 					"#################################",
-					"All tests passed in REPLAYING mode.", "#################################",
+					"All tests passed in REPLAYING mode.",
+					"#################################",
 				},
 				"\n",
 			),
@@ -93,6 +99,8 @@ func TestFormatVCRCassettesUpdateReplaying(t *testing.T) {
 				[]string{
 					"#################################",
 					"Tests Analytics",
+					"#################################",
+					"",
 					"Total tests: 5",
 					"Passed tests: 2",
 					"Skipped tests: 1",
@@ -160,6 +168,7 @@ func TestFormatVCRCassettesUpdateRecording(t *testing.T) {
 				[]string{
 					"#################################",
 					"RECORDING Tests Report",
+					"#################################",
 					"",
 					"",
 					"Tests passed during RECORDING mode:",
@@ -173,7 +182,7 @@ func TestFormatVCRCassettesUpdateRecording(t *testing.T) {
 					"",
 					"",
 					"",
-					"Errors occurred during RECORDING mode.",
+					"Errors occurred during RECORDING mode: some error.",
 				},
 				"\n",
 			),
@@ -190,6 +199,7 @@ func TestFormatVCRCassettesUpdateRecording(t *testing.T) {
 				[]string{
 					"#################################",
 					"RECORDING Tests Report",
+					"#################################",
 					"",
 					"",
 					"Tests passed during RECORDING mode:",
@@ -219,6 +229,7 @@ func TestFormatVCRCassettesUpdateRecording(t *testing.T) {
 				[]string{
 					"#################################",
 					"RECORDING Tests Report",
+					"#################################",
 					"",
 					"",
 					"Tests passed during RECORDING mode:",
@@ -264,6 +275,7 @@ func TestFormatVCRCassettesUpdateRecording(t *testing.T) {
 				[]string{
 					"#################################",
 					"RECORDING Tests Report",
+					"#################################",
 					"",
 					"",
 					"Tests passed during RECORDING mode:",
@@ -371,7 +383,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 					}},
 					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/recording_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/recording/"}, map[string]string(nil)},
 					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/recording/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
-					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta", "gs://ci-vcr-cassettes/beta/fixtures/"}, map[string]string(nil)},
+					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://ci-vcr-cassettes/beta/fixtures/"}, map[string]string(nil)},
 				},
 			},
 		},

.ci/magician/github/membership_data.go

@@ -35,6 +35,11 @@ var (
 		// 	startDate: newDate(2024, 3, 28, pdtLoc),
 		// 	endDate:   newDate(2024, 4, 2, pdtLoc),
 		// },
+		{
+			id:        "BBBmau",
+			startDate: newDate(2024, 9, 26, pdtLoc),
+			endDate:   newDate(2024, 10, 2, pdtLoc),
+		},
 		{
 			id:        "hao-nan-li",
 			startDate: newDate(2024, 9, 24, pdtLoc),
@@ -45,15 +50,10 @@ var (
 			startDate: newDate(2024, 4, 30, pdtLoc),
 			endDate:   newDate(2024, 7, 31, pdtLoc),
 		},
-		{
-			id:        "SarahFrench",
-			startDate: newDate(2024, 8, 2, bstLoc),
-			endDate:   newDate(2024, 8, 6, bstLoc),
-		},
 		{
 			id:        "shuyama1",
-			startDate: newDate(2024, 5, 22, pdtLoc),
-			endDate:   newDate(2024, 5, 28, pdtLoc),
+			startDate: newDate(2024, 9, 26, pdtLoc),
+			endDate:   newDate(2024, 10, 4, pdtLoc),
 		},
 		{
 			id:        "melinath",
@@ -95,5 +95,10 @@ var (
 			startDate: newDate(2024, 9, 13, pdtLoc),
 			endDate:   newDate(2024, 9, 20, pdtLoc),
 		},
+		{
+			id:        "SarahFrench",
+			startDate: newDate(2024, 9, 20, bstLoc),
+			endDate:   newDate(2024, 9, 23, bstLoc),
+		},
 	}
 )

GNUmakefile

@@ -95,6 +95,11 @@ tgc:
 		bundle exec compiler -e terraform -f tgc -v beta -o $(OUTPUT_PATH)/tfplan2cai $(mmv1_compile);\
 		bundle exec compiler -e terraform -f tgc_cai2hcl -v beta -o $(OUTPUT_PATH)/cai2hcl $(mmv1_compile);\
 
+tgc-go:
+	cd mmv1;\
+		go run . --version beta --provider  tgc --output $(OUTPUT_PATH)/tfplan2cai;\
+		go run . --version beta --provider  tgc_cai2hcl --output $(OUTPUT_PATH)/cai2hcl;\
+
 tf-oics:
 	cd mmv1;\
 		bundle;\

docs/content/best-practices/_index.md

@@ -67,8 +67,8 @@ When adding a new `labels` field, please make the changes below to support the n
 1. Use the type `KeyValueLabels` for the standard resource `labels` field. The standard resource `labels` field could be the top level `labels` field or the nested `labels` field inside the top level `metadata` field. Don't add `default_from_api: true` to this field or don't use this type for other `labels` fields in the resource. `KeyValueLabels` will add all of changes required for the new model automatically.
 
 ```yaml
-- !ruby/object:Api::Type::KeyValueLabels
-   name: 'labels'
+ - name: 'labels'
+   type: KeyValueLabels
    description: |
    The labels associated with this dataset. You can use these to
    organize and group your datasets.
@@ -155,9 +155,10 @@ When adding a new `annotations` field, please make the changes below below to su
 1. Use the type `KeyValueAnnotations` for the standard resource `annotations` field. The standard resource `annotations` field could be the top level `annotations` field or the nested `annotations` field inside the top level `metadata` field. Don't add `default_from_api: true` to this field or don't use this type for other `annotations` fields in the resource. `KeyValueAnnotations` will add all of changes required for the new model automatically.
 
 ```yaml
-- !ruby/object:Api::Type::KeyValueAnnotations
-   name: 'annotations'
-   description: 'Client-specified annotations. This is distinct from labels.'
+- name: 'annotations'
+  type: KeyValueAnnotations
+  description: |
+   Client-specified annotations. This is distinct from labels.
 ```
 2. In the handwritten acceptance tests, add `annotations` to `ImportStateVerifyIgnore` if `annotations` field is in the configuration.
 

docs/content/develop/add-handwritten-datasource.md

@@ -24,7 +24,7 @@ a new datasource there are 5 steps to doing so.
 1. Add Schema and Read operation implementation
    - If there is `labels` field with type `KeyValueLabels` in the corresponding resource, in the datasource Read operation implementation, after the resource read method, call the function `tpgresource.SetDataSourceLabels(d)` to make `labels` and `terraform_labels` have all of the labels on the resource.
    - If there is `annotations` field with type `KeyValueAnnotations` in the corresponding resource, in the datasource Read operation implementation, after the resource read method, call the function `tpgresource.SetDataSourceAnnotations(d)` to make `annotations` have all of the annotations on the resource.
-1. Register the datasource to `handwrittenDatasources` in [`magic-modules/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb`](https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb)
+1. Register the datasource to `handwrittenDatasources` in [`magic-modules/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.tmpl`](https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.tmpl)
 1. Implement a test which will create and resources and read the corresponding
   datasource
 1. [Add documentation](#add-documentation)

docs/content/develop/breaking-changes/make-a-breaking-change.md

@@ -91,11 +91,11 @@ provider at runtime as well as in documentation.
 Set `deprecation_message` on the field. For example:
 
 ```yaml
-- !ruby/object:Api::Type::String
-  name: 'apiFieldName'
+- name: 'apiFieldName'
+  type: String
   description: |
     MULTILINE_FIELD_DESCRIPTION
-  deprecation_message: "`api_field_name` is deprecated and will be removed in a future major release. Use `other_field_name` instead."
+  deprecation_message: '`api_field_name` is deprecated and will be removed in a future major release. Use `other_field_name` instead.'
 ```
 
 Replace the second sentence with an appropriate short description of the replacement path and/or the reason for

docs/content/develop/client-side-fields.md

@@ -21,8 +21,8 @@ Instead of adding the field in `parameters` or `properties`, use a section calle
 Example:
 ```yaml
 virtual_fields:
-  - !ruby/object:Api::Type::Boolean
-    name: 'deletion_protection'
+  - name: 'deletion_protection'
+    type: Boolean
     default_value: true
     description: |
       Whether Terraform will be prevented from destroying the CertificateAuthority.