Merge branch 'GoogleCloudPlatform:main' into logscope

Author: tyh2333

.ci/infra/terraform/main.tf

@@ -423,6 +423,21 @@ resource "google_project_iam_member" "aiplatform_agent_encrypter_decrypter" {
   member  = "serviceAccount:service-${google_project.proj.number}@gcp-sa-aiplatform.iam.gserviceaccount.com"
 }
 
+# TestAccComputeInstance_confidentialHyperDiskBootDisk
+resource "google_project_iam_member" "compute_default_sa_encrypter_decrypter" {
+  project = google_project.proj.project_id
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member  = "serviceAccount:${google_project.proj.number}[email protected]"
+}
+
+# TestAccComputeInstance_confidentialHyperDiskBootDisk
+resource "google_project_iam_member" "compute_agent_encrypter_decrypter" {
+  project = google_project.proj.project_id
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member  = "serviceAccount:service-${google_project.proj.number}@compute-system.iam.gserviceaccount.com"
+}
+
+
 data "google_organization" "org2" {
   organization = var.org2_id
 }

.ci/magician/cmd/check_cassettes.go

@@ -64,7 +64,7 @@ var checkCassettesCmd = &cobra.Command{
 
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr)
 
-		vt, err := vcr.NewTester(env, rnr)
+		vt, err := vcr.NewTester(env, "vcr-check-cassettes", "ci-vcr-cassettes", rnr)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}
@@ -110,7 +110,10 @@ func execCheckCassettes(commit string, vt *vcr.Tester, ctlr *source.Controller)
 	if err != nil {
 		fmt.Println("Error running VCR: ", err)
 	}
-	if err := vt.UploadLogs("vcr-check-cassettes", "", "", false, false, vcr.Replaying, provider.Beta); err != nil {
+	if err := vt.UploadLogs(vcr.UploadLogsOptions{
+		Mode:    vcr.Replaying,
+		Version: provider.Beta,
+	}); err != nil {
 		return fmt.Errorf("error uploading logs: %w", err)
 	}
 	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)

.ci/magician/cmd/test_terraform_vcr.go

@@ -120,7 +120,7 @@ var testTerraformVCRCmd = &cobra.Command{
 		}
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr)
 
-		vt, err := vcr.NewTester(env, rnr)
+		vt, err := vcr.NewTester(env, "ci-vcr-logs", "ci-vcr-cassettes", rnr)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}
@@ -195,7 +195,12 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 		testState = "failure"
 	}
 
-	if err := vt.UploadLogs("ci-vcr-logs", prNumber, buildID, false, false, vcr.Replaying, provider.Beta); err != nil {
+	if err := vt.UploadLogs(vcr.UploadLogsOptions{
+		PRNumber: prNumber,
+		BuildID:  buildID,
+		Mode:     vcr.Replaying,
+		Version:  provider.Beta,
+	}); err != nil {
 		return fmt.Errorf("error uploading replaying logs: %w", err)
 	}
 
@@ -256,11 +261,17 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 			testState = "success"
 		}
 
-		if err := vt.UploadCassettes("ci-vcr-cassettes", prNumber, provider.Beta); err != nil {
+		if err := vt.UploadCassettes(prNumber, provider.Beta); err != nil {
 			return fmt.Errorf("error uploading cassettes: %w", err)
 		}
 
-		if err := vt.UploadLogs("ci-vcr-logs", prNumber, buildID, true, false, vcr.Recording, provider.Beta); err != nil {
+		if err := vt.UploadLogs(vcr.UploadLogsOptions{
+			PRNumber: prNumber,
+			BuildID:  buildID,
+			Parallel: true,
+			Mode:     vcr.Recording,
+			Version:  provider.Beta,
+		}); err != nil {
 			return fmt.Errorf("error uploading recording logs: %w", err)
 		}
 
@@ -283,9 +294,17 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 				testState = "failure"
 			}
 
-			if err := vt.UploadLogs("ci-vcr-logs", prNumber, buildID, true, true, vcr.Replaying, provider.Beta); err != nil {
+			if err := vt.UploadLogs(vcr.UploadLogsOptions{
+				PRNumber:       prNumber,
+				BuildID:        buildID,
+				Parallel:       true,
+				AfterRecording: true,
+				Mode:           vcr.Replaying,
+				Version:        provider.Beta,
+			}); err != nil {
 				return fmt.Errorf("error uploading recording logs: %w", err)
 			}
+
 		}
 
 		hasTerminatedTests := (len(recordingResult.PassedTests) + len(recordingResult.FailedTests)) < len(replayingResult.FailedTests)

.ci/magician/cmd/vcr_cassette_update.go

@@ -90,7 +90,7 @@ var vcrCassetteUpdateCmd = &cobra.Command{
 		}
 		ctlr := source.NewController(env["GOPATH"], "hashicorp", env["GITHUB_TOKEN_CLASSIC"], rnr)
 
-		vt, err := vcr.NewTester(env, rnr)
+		vt, err := vcr.NewTester(env, "", "ci-vcr-cassettes", rnr)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}

.ci/magician/cmd/vcr_cassette_update_test.go

@@ -389,7 +389,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 			ctlr := source.NewController("gopath", "hashicorp", "token", rnr)
 			vt, err := vcr.NewTester(map[string]string{
 				"SA_KEY": "sa_key",
-			}, rnr)
+			}, "", "ci-vcr-cassettes", rnr)
 			if err != nil {
 				t.Fatalf("Failed to create new tester: %v", err)
 			}

.ci/magician/vcr/tester.go

@@ -48,13 +48,15 @@ type logKey struct {
 }
 
 type Tester struct {
-	env           map[string]string           // shared environment variables for running tests
-	rnr           ExecRunner                  // for running commands and manipulating files
-	baseDir       string                      // the directory in which this tester was created
-	saKeyPath     string                      // where sa_key.json is relative to baseDir
-	cassettePaths map[provider.Version]string // where cassettes are relative to baseDir by version
-	logPaths      map[logKey]string           // where logs are relative to baseDir by version and mode
-	repoPaths     map[provider.Version]string // relative paths of already cloned repos by version
+	env            map[string]string           // shared environment variables for running tests
+	rnr            ExecRunner                  // for running commands and manipulating files
+	logBucket      string                      // GCS bucket name to store logs
+	cassetteBucket string                      // GCS bucket name to store cassettes
+	baseDir        string                      // the directory in which this tester was created
+	saKeyPath      string                      // where sa_key.json is relative to baseDir
+	cassettePaths  map[provider.Version]string // where cassettes are relative to baseDir by version
+	logPaths       map[logKey]string           // where logs are relative to baseDir by version and mode
+	repoPaths      map[provider.Version]string // relative paths of already cloned repos by version
 }
 
 const accTestParallelism = 32
@@ -67,19 +69,24 @@ var testResultsExpression = regexp.MustCompile(`(?m:^--- (PASS|FAIL|SKIP): (Test
 var testPanicExpression = regexp.MustCompile(`^panic: .*`)
 
 // Create a new tester in the current working directory and write the service account key file.
-func NewTester(env map[string]string, rnr ExecRunner) (*Tester, error) {
-	saKeyPath := "sa_key.json"
-	if err := rnr.WriteFile(saKeyPath, env["SA_KEY"]); err != nil {
-		return nil, err
+func NewTester(env map[string]string, logBucket, cassetteBucket string, rnr ExecRunner) (*Tester, error) {
+	var saKeyPath string
+	if saKeyVal, ok := env["SA_KEY"]; ok {
+		saKeyPath = "sa_key.json"
+		if err := rnr.WriteFile(saKeyPath, saKeyVal); err != nil {
+			return nil, err
+		}
 	}
 	return &Tester{
-		env:           env,
-		rnr:           rnr,
-		baseDir:       rnr.GetCWD(),
-		saKeyPath:     saKeyPath,
-		cassettePaths: make(map[provider.Version]string, provider.NumVersions),
-		logPaths:      make(map[logKey]string, provider.NumVersions*numModes),
-		repoPaths:     make(map[provider.Version]string, provider.NumVersions),
+		env:            env,
+		rnr:            rnr,
+		logBucket:      logBucket,
+		cassetteBucket: cassetteBucket,
+		baseDir:        rnr.GetCWD(),
+		saKeyPath:      saKeyPath,
+		cassettePaths:  make(map[provider.Version]string, provider.NumVersions),
+		logPaths:       make(map[logKey]string, provider.NumVersions*numModes),
+		repoPaths:      make(map[provider.Version]string, provider.NumVersions),
 	}, nil
 }
 
@@ -98,19 +105,19 @@ func (vt *Tester) FetchCassettes(version provider.Version, baseBranch, prNumber
 	vt.rnr.Mkdir(cassettePath)
 	if baseBranch != "FEATURE-BRANCH-major-release-6.0.0" {
 		// pull main cassettes (major release uses branch specific casssettes as primary ones)
-		bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%sfixtures/*", version.BucketPath())
+		bucketPath := fmt.Sprintf("gs://%s/%sfixtures/*", vt.cassetteBucket, version.BucketPath())
 		if err := vt.fetchBucketPath(bucketPath, cassettePath); err != nil {
 			fmt.Println("Error fetching cassettes: ", err)
 		}
 	}
 	if baseBranch != "main" {
-		bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%srefs/branches/%s/fixtures/*", version.BucketPath(), baseBranch)
+		bucketPath := fmt.Sprintf("gs://%s/%srefs/branches/%s/fixtures/*", vt.cassetteBucket, version.BucketPath(), baseBranch)
 		if err := vt.fetchBucketPath(bucketPath, cassettePath); err != nil {
 			fmt.Println("Error fetching cassettes: ", err)
 		}
 	}
 	if prNumber != "" {
-		bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%srefs/heads/auto-pr-%s/fixtures/*", version.BucketPath(), prNumber)
+		bucketPath := fmt.Sprintf("gs://%s/%srefs/heads/auto-pr-%s/fixtures/*", vt.cassetteBucket, version.BucketPath(), prNumber)
 		if err := vt.fetchBucketPath(bucketPath, cassettePath); err != nil {
 			fmt.Println("Error fetching cassettes: ", err)
 		}
@@ -396,49 +403,93 @@ func (vt *Tester) getLogPath(mode Mode, version provider.Version) (string, error
 	return logPath, nil
 }
 
-func (vt *Tester) UploadLogs(logBucket, prNumber, buildID string, parallel, afterRecording bool, mode Mode, version provider.Version) error {
-	bucketPath := fmt.Sprintf("gs://%s/%s/", logBucket, version)
-	if prNumber != "" {
-		bucketPath += fmt.Sprintf("refs/heads/auto-pr-%s/", prNumber)
+// UploadLogsOptions defines options for uploading logs.
+type UploadLogsOptions struct {
+	PRNumber       string
+	BuildID        string
+	Parallel       bool
+	AfterRecording bool
+	Mode           Mode
+	Version        provider.Version
+}
+
+// UploadLogs uploads logs to Google Cloud Storage.
+func (vt *Tester) UploadLogs(opts UploadLogsOptions) error {
+	bucketPath := fmt.Sprintf("gs://%s/%s/", vt.logBucket, opts.Version)
+	if opts.PRNumber != "" {
+		bucketPath += fmt.Sprintf("refs/heads/auto-pr-%s/", opts.PRNumber)
 	}
-	if buildID != "" {
-		bucketPath += fmt.Sprintf("artifacts/%s/", buildID)
+	if opts.BuildID != "" {
+		bucketPath += fmt.Sprintf("artifacts/%s/", opts.BuildID)
 	}
-	lgky := logKey{mode, version}
+	lgky := logKey{opts.Mode, opts.Version}
 	logPath, ok := vt.logPaths[lgky]
 	if !ok {
-		return fmt.Errorf("no log path found for mode %s and version %s", mode.Lower(), version)
+		return fmt.Errorf("no log path found for mode %s and version %s", opts.Mode.Lower(), opts.Version)
+	}
+	args := []string{
+		"-h",
+		"Content-Type:text/plain",
+		"-q",
+		"cp",
+		"-r",
+		filepath.Join(vt.baseDir, "testlogs", fmt.Sprintf("%s_test.log", opts.Mode.Lower())),
+		bucketPath + "build-log/",
 	}
-	args := []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", filepath.Join(vt.baseDir, "testlogs", fmt.Sprintf("%s_test.log", mode.Lower())), bucketPath + "build-log/"}
 	fmt.Println("Uploading build log:\n", "gsutil", strings.Join(args, " "))
 	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
 		fmt.Println("Error uploading build log: ", err)
 	}
 	var suffix string
-	if afterRecording {
+	if opts.AfterRecording {
 		suffix = "_after_recording"
 	}
-	if parallel {
-		args := []string{"-h", "Content-Type:text/plain", "-m", "-q", "cp", "-r", filepath.Join(vt.baseDir, "testlogs", mode.Lower()+"_build", "*"), fmt.Sprintf("%sbuild-log/%s_build%s/", bucketPath, mode.Lower(), suffix)}
+	if opts.Parallel {
+		args := []string{
+			"-h",
+			"Content-Type:text/plain",
+			"-m",
+			"-q",
+			"cp",
+			"-r",
+			filepath.Join(vt.baseDir, "testlogs", opts.Mode.Lower()+"_build", "*"),
+			fmt.Sprintf("%sbuild-log/%s_build%s/", bucketPath, opts.Mode.Lower(), suffix),
+		}
 		fmt.Println("Uploading build logs:\n", "gsutil", strings.Join(args, " "))
 		if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
 			fmt.Println("Error uploading build logs: ", err)
 		}
 	}
-	args = []string{"-h", "Content-Type:text/plain", "-m", "-q", "cp", "-r", filepath.Join(logPath, "*"), fmt.Sprintf("%s%s%s/", bucketPath, mode.Lower(), suffix)}
+	args = []string{
+		"-h",
+		"Content-Type:text/plain",
+		"-m",
+		"-q",
+		"cp",
+		"-r",
+		filepath.Join(logPath, "*"),
+		fmt.Sprintf("%s%s%s/", bucketPath, opts.Mode.Lower(), suffix),
+	}
 	fmt.Println("Uploading logs:\n", "gsutil", strings.Join(args, " "))
 	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
 		fmt.Println("Error uploading logs: ", err)
+		vt.printLogs(logPath)
 	}
 	return nil
 }
 
-func (vt *Tester) UploadCassettes(logBucket, prNumber string, version provider.Version) error {
+func (vt *Tester) UploadCassettes(prNumber string, version provider.Version) error {
 	cassettePath, ok := vt.cassettePaths[version]
 	if !ok {
 		return fmt.Errorf("no cassettes found for version %s", version)
 	}
-	args := []string{"-m", "-q", "cp", filepath.Join(cassettePath, "*"), fmt.Sprintf("gs://%s/%s/refs/heads/auto-pr-%s/fixtures/", logBucket, version, prNumber)}
+	args := []string{
+		"-m",
+		"-q",
+		"cp",
+		filepath.Join(cassettePath, "*"),
+		fmt.Sprintf("gs://%s/%s/refs/heads/auto-pr-%s/fixtures/", vt.cassetteBucket, version, prNumber),
+	}
 	fmt.Println("Uploading cassettes:\n", "gsutil", strings.Join(args, " "))
 	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
 		fmt.Println("Error uploading cassettes: ", err)

mmv1/api/resource.go

@@ -1192,8 +1192,7 @@ func (r Resource) ExtractIdentifiers(url string) []string {
 	return result
 }
 
-// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
-func (r Resource) RawImportIdFormatsFromIam() []string {
+func (r Resource) IamImportFormats() []string {
 	var importFormat []string
 
 	if r.IamPolicy != nil {
@@ -1202,8 +1201,12 @@ func (r Resource) RawImportIdFormatsFromIam() []string {
 	if len(importFormat) == 0 {
 		importFormat = r.ImportFormat
 	}
+	return importFormat
+}
 
-	return ImportIdFormats(importFormat, r.Identity, r.BaseUrl)
+// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
+func (r Resource) RawImportIdFormatsFromIam() []string {
+	return ImportIdFormats(r.IamImportFormats(), r.Identity, r.BaseUrl)
 }
 
 // For example, projects/(?P<project>[^/]+)/schemas/(?P<schema>[^/]+)", "(?P<project>[^/]+)/(?P<schema>[^/]+)", "(?P<schema>[^/]+)
@@ -1668,3 +1671,34 @@ func (r Resource) CaiApiVersion(productBackendName, caiProductBaseUrl string) st
 	}
 	return ""
 }
+
+// For example: the uri "projects/{{project}}/schemas/{{name}}"
+// The paramerter is "schema" as "project" is not returned.
+func (r Resource) CaiIamResourceParams() []string {
+	resourceUri := strings.ReplaceAll(r.IamResourceUri(), "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+
+	return google.Reject(r.ExtractIdentifiers(resourceUri), func(param string) bool {
+		return param == "project"
+	})
+}
+
+// Gets the Cai IAM asset name template
+// For example: //monitoring.googleapis.com/v3/projects/{{project}}/services/{{service_id}}
+func (r Resource) CaiIamAssetNameTemplate(productBackendName string) string {
+	iamImportFormat := r.IamImportFormats()
+	if len(iamImportFormat) > 0 {
+		name := strings.ReplaceAll(iamImportFormat[0], "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+		name = strings.ReplaceAll(name, "%", "")
+		return fmt.Sprintf("//%s.googleapis.com/%s", productBackendName, name)
+	}
+
+	caiBaseUrl := r.CaiBaseUrl
+
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.SelfLink
+	}
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.BaseUrl
+	}
+	return fmt.Sprintf("//%s.googleapis.com/%s/{{%s}}", productBackendName, caiBaseUrl, r.IamParentResourceName())
+}

mmv1/google/slice_utils.go

@@ -39,3 +39,19 @@ func Reject[T any](S []T, test func(T) bool) (ret []T) {
 func Concat[T any](S1 []T, S2 []T) (ret []T) {
 	return append(S1, S2...)
 }
+
+// difference returns the elements in `S1` that aren't in `S2`.
+func Diff(S1, S2 []string) []string {
+	var ret []string
+	mb := make(map[string]bool, len(S2))
+	for _, x := range S2 {
+		mb[x] = true
+	}
+
+	for _, x := range S1 {
+		if _, found := mb[x]; !found {
+			ret = append(ret, x)
+		}
+	}
+	return ret
+}