Rewrite tgc provider and resource converter template (#11706)

Author: zli82016

mmv1/api/resource.go

@@ -1601,3 +1601,70 @@ func (r Resource) StateUpgradersCount() []int {
 	}
 	return nums
 }
+
+func (r Resource) CaiProductBaseUrl() string {
+	version := r.ProductMetadata.VersionObjOrClosest(r.TargetVersionName)
+	baseUrl := version.CaiBaseUrl
+	if baseUrl == "" {
+		baseUrl = version.BaseUrl
+	}
+	return baseUrl
+}
+
+// Returns the Cai product backend name from the version base url
+// base_url: https://accessapproval.googleapis.com/v1/ -> accessapproval
+func (r Resource) CaiProductBackendName(caiProductBaseUrl string) string {
+	backendUrl := strings.Split(strings.Split(caiProductBaseUrl, "://")[1], ".googleapis.com")[0]
+	return strings.ToLower(backendUrl)
+}
+
+// Gets the Cai asset name template, which could include version
+// For example: //monitoring.googleapis.com/v3/projects/{{project}}/services/{{service_id}}
+func (r Resource) rawCaiAssetNameTemplate(productBackendName string) string {
+	caiBaseUrl := ""
+	if r.CaiBaseUrl != "" {
+		caiBaseUrl = fmt.Sprintf("%s/{{name}}", r.CaiBaseUrl)
+	}
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.SelfLink
+	}
+	if caiBaseUrl == "" {
+		caiBaseUrl = fmt.Sprintf("%s/{{name}}", r.BaseUrl)
+	}
+	return fmt.Sprintf("//%s.googleapis.com/%s", productBackendName, caiBaseUrl)
+}
+
+// Gets the Cai asset name template, which doesn't include version
+// For example: //monitoring.googleapis.com/projects/{{project}}/services/{{service_id}}
+func (r Resource) CaiAssetNameTemplate(productBackendName string) string {
+	template := r.rawCaiAssetNameTemplate(productBackendName)
+	versionRegex, err := regexp.Compile(`\/(v\d[^\/]*)\/`)
+	if err != nil {
+		log.Fatalf("Cannot compile the regular expression: %v", err)
+	}
+
+	return versionRegex.ReplaceAllString(template, "/")
+}
+
+// Gets the Cai API version
+func (r Resource) CaiApiVersion(productBackendName, caiProductBaseUrl string) string {
+	template := r.rawCaiAssetNameTemplate(productBackendName)
+
+	versionRegex, err := regexp.Compile(`\/(v\d[^\/]*)\/`)
+	if err != nil {
+		log.Fatalf("Cannot compile the regular expression: %v", err)
+	}
+
+	apiVersion := strings.ReplaceAll(versionRegex.FindString(template), "/", "")
+	if apiVersion != "" {
+		return apiVersion
+	}
+
+	splits := strings.Split(caiProductBaseUrl, "/")
+	for i := 0; i < len(splits); i++ {
+		if splits[len(splits)-1-i] != "" {
+			return splits[len(splits)-1-i]
+		}
+	}
+	return ""
+}

mmv1/api/type.go

@@ -422,7 +422,8 @@ func (t *Type) GetPrefix() string {
 	if t.Prefix == "" {
 		if t.ParentMetadata == nil {
 			nestedPrefix := ""
-			if t.ResourceMetadata.NestedQuery != nil {
+			// TODO: Use the nestedPrefix for tgc provider to be consistent with terraform provider
+			if t.ResourceMetadata.NestedQuery != nil && t.ResourceMetadata.Compiler != "terraformgoogleconversion-codegen" {
 				nestedPrefix = "Nested"
 			}
 

mmv1/main.go

@@ -36,6 +36,8 @@ var doNotGenerateCode = flag.Bool("no-code", false, "do not generate code")
 
 var doNotGenerateDocs = flag.Bool("no-docs", false, "do not generate docs")
 
+var forceProvider = flag.String("provider", "", "optional provider name. If specified, a non-default provider will be used.")
+
 // Example usage: --yaml
 var yamlMode = flag.Bool("yaml", false, "copy text over from ruby yaml to go yaml")
 
@@ -110,6 +112,7 @@ func main() {
 	startTime := time.Now()
 	log.Printf("Generating MM output to '%s'", *outputPath)
 	log.Printf("Using %s version", *version)
+	log.Printf("Using %s provider", *forceProvider)
 
 	// Building compute takes a long time and can't be parallelized within the product
 	// so lets build it first
@@ -120,7 +123,7 @@ func main() {
 		return false
 	})
 
-	var providerToGenerate *provider.Terraform
+	var providerToGenerate provider.Provider
 	var productsForVersion []*api.Product
 
 	ch := make(chan string, len(allProductFiles))
@@ -143,8 +146,7 @@ func main() {
 	// In order to only copy/compile files once per provider this must be called outside
 	// of the products loop. This will get called with the provider from the final iteration
 	// of the loop
-	providerToGenerate = provider.NewTerraform(productsForVersion[0], *version, startTime)
-
+	providerToGenerate = setProvider(*forceProvider, *version, productsForVersion[0], startTime)
 	providerToGenerate.CopyCommonFiles(*outputPath, generateCode, generateDocs)
 
 	log.Printf("Compiling common files for terraform")
@@ -155,7 +157,7 @@ func main() {
 	}
 }
 
-func GenerateProduct(productChannel chan string, providerToGenerate *provider.Terraform, productsForVersion *[]*api.Product, startTime time.Time, productsToGenerate []string, resourceToGenerate string, generateCode, generateDocs bool) {
+func GenerateProduct(productChannel chan string, providerToGenerate provider.Provider, productsForVersion *[]*api.Product, startTime time.Time, productsToGenerate []string, resourceToGenerate string, generateCode, generateDocs bool) {
 
 	defer wg.Done()
 	productName := <-productChannel
@@ -215,8 +217,7 @@ func GenerateProduct(productChannel chan string, providerToGenerate *provider.Te
 		productApi.Objects = resources
 		productApi.Validate()
 
-		// TODO rewrite: set other providers via flag
-		providerToGenerate = provider.NewTerraform(productApi, *version, startTime)
+		providerToGenerate = setProvider(*forceProvider, *version, productApi, startTime)
 
 		*productsForVersion = append(*productsForVersion, productApi)
 
@@ -229,3 +230,13 @@ func GenerateProduct(productChannel chan string, providerToGenerate *provider.Te
 		providerToGenerate.Generate(*outputPath, productName, resourceToGenerate, generateCode, generateDocs)
 	}
 }
+
+// Sets provider via flag
+func setProvider(forceProvider, version string, productApi *api.Product, startTime time.Time) provider.Provider {
+	switch forceProvider {
+	case "tgc":
+		return provider.NewTerraformGoogleConversion(productApi, version, startTime)
+	default:
+		return provider.NewTerraform(productApi, version, startTime)
+	}
+}

mmv1/provider/provider.go

@@ -0,0 +1,45 @@
+package provider
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
+)
+
+type Provider interface {
+	Generate(string, string, string, bool, bool)
+	CopyCommonFiles(outputFolder string, generateCode, generateDocs bool)
+	CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string)
+}
+
+// Shared constants and functions among the providers
+
+const TERRAFORM_PROVIDER_GA = "github.com/hashicorp/terraform-provider-google"
+const TERRAFORM_PROVIDER_BETA = "github.com/hashicorp/terraform-provider-google-beta"
+const TERRAFORM_PROVIDER_PRIVATE = "internal/terraform-next"
+const RESOURCE_DIRECTORY_GA = "google"
+const RESOURCE_DIRECTORY_BETA = "google-beta"
+const RESOURCE_DIRECTORY_PRIVATE = "google-private"
+
+// # TODO(nelsonjr): Review all object interfaces and move to private methods
+// # that should not be exposed outside the object hierarchy.
+func ProviderName(t Provider) string {
+	return reflect.TypeOf(t).Name()
+}
+
+func ImportPathFromVersion(t Provider, v string) string {
+	var tpg, dir string
+	switch v {
+	case "ga":
+		tpg = TERRAFORM_PROVIDER_GA
+		dir = RESOURCE_DIRECTORY_GA
+	case "beta":
+		tpg = TERRAFORM_PROVIDER_BETA
+		dir = RESOURCE_DIRECTORY_BETA
+	default:
+		tpg = TERRAFORM_PROVIDER_PRIVATE
+		dir = RESOURCE_DIRECTORY_PRIVATE
+	}
+	return fmt.Sprintf("%s/%s", tpg, dir)
+}

mmv1/provider/template_data.go

@@ -168,6 +168,15 @@ func (td *TemplateData) GenerateSweeperFile(filePath string, resource api.Resour
 	td.GenerateFile(filePath, templatePath, resource, false, templates...)
 }
 
+func (td *TemplateData) GenerateTGCResourceFile(filePath string, resource api.Resource) {
+	templatePath := "templates/tgc/resource_converter.go.tmpl"
+	templates := []string{
+		templatePath,
+		"templates/terraform/expand_property_method.go.tmpl",
+	}
+	td.GenerateFile(filePath, templatePath, resource, true, templates...)
+}
+
 func (td *TemplateData) GenerateFile(filePath, templatePath string, input any, goFormat bool, templates ...string) {
 	// log.Printf("Generating %s", filePath)
 

mmv1/provider/terraform.go

@@ -24,7 +24,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"reflect"
 	"slices"
 	"strings"
 	"time"
@@ -35,13 +34,6 @@ import (
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
 )
 
-const TERRAFORM_PROVIDER_GA = "github.com/hashicorp/terraform-provider-google"
-const TERRAFORM_PROVIDER_BETA = "github.com/hashicorp/terraform-provider-google-beta"
-const TERRAFORM_PROVIDER_PRIVATE = "internal/terraform-next"
-const RESOURCE_DIRECTORY_GA = "google"
-const RESOURCE_DIRECTORY_BETA = "google-beta"
-const RESOURCE_DIRECTORY_PRIVATE = "google-private"
-
 type Terraform struct {
 	ResourceCount int
 
@@ -58,7 +50,7 @@ type Terraform struct {
 	StartTime time.Time
 }
 
-func NewTerraform(product *api.Product, versionName string, startTime time.Time) *Terraform {
+func NewTerraform(product *api.Product, versionName string, startTime time.Time) Terraform {
 	t := Terraform{
 		ResourceCount:     0,
 		IAMResourceCount:  0,
@@ -70,14 +62,14 @@ func NewTerraform(product *api.Product, versionName string, startTime time.Time)
 
 	t.Product.SetPropertiesBasedOnVersion(&t.Version)
 	for _, r := range t.Product.Objects {
-		r.SetCompiler(t.providerName())
-		r.ImportPath = t.ImportPathFromVersion(versionName)
+		r.SetCompiler(ProviderName(t))
+		r.ImportPath = ImportPathFromVersion(t, versionName)
 	}
 
-	return &t
+	return t
 }
 
-func (t *Terraform) Generate(outputFolder, productPath, resourceToGenerate string, generateCode, generateDocs bool) {
+func (t Terraform) Generate(outputFolder, productPath, resourceToGenerate string, generateCode, generateDocs bool) {
 	if err := os.MkdirAll(outputFolder, os.ModePerm); err != nil {
 		log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err))
 	}
@@ -273,7 +265,7 @@ func (t *Terraform) FullResourceName(object api.Resource) string {
 }
 
 func (t Terraform) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) {
-	log.Printf("Copying common files for %s", t.providerName())
+	log.Printf("Copying common files for %s", ProviderName(t))
 
 	files := t.getCommonCopyFiles(t.TargetVersionName, generateCode, generateDocs)
 	t.CopyFileList(outputFolder, files)
@@ -565,8 +557,8 @@ func (t Terraform) replaceImportPath(outputFolder, target string) {
 
 	data := string(sourceByte)
 
-	gaImportPath := t.ImportPathFromVersion("ga")
-	betaImportPath := t.ImportPathFromVersion("beta")
+	gaImportPath := ImportPathFromVersion(t, "ga")
+	betaImportPath := ImportPathFromVersion(t, "beta")
 
 	if strings.Contains(data, betaImportPath) {
 		log.Fatalf("Importing a package from module %s is not allowed in file %s. Please import a package from module %s.", betaImportPath, filepath.Base(target), gaImportPath)
@@ -607,22 +599,6 @@ func (t Terraform) replaceImportPath(outputFolder, target string) {
 	}
 }
 
-func (t Terraform) ImportPathFromVersion(v string) string {
-	var tpg, dir string
-	switch v {
-	case "ga":
-		tpg = TERRAFORM_PROVIDER_GA
-		dir = RESOURCE_DIRECTORY_GA
-	case "beta":
-		tpg = TERRAFORM_PROVIDER_BETA
-		dir = RESOURCE_DIRECTORY_BETA
-	default:
-		tpg = TERRAFORM_PROVIDER_PRIVATE
-		dir = RESOURCE_DIRECTORY_PRIVATE
-	}
-	return fmt.Sprintf("%s/%s", tpg, dir)
-}
-
 func (t Terraform) ProviderFromVersion() string {
 	var dir string
 	switch t.TargetVersionName {
@@ -708,12 +684,6 @@ func (t *Terraform) generateResourcesForVersion(products []*api.Product) {
 	}
 }
 
-// # TODO(nelsonjr): Review all object interfaces and move to private methods
-// # that should not be exposed outside the object hierarchy.
-func (t Terraform) providerName() string {
-	return reflect.TypeOf(t).Name()
-}
-
 // # Adapted from the method used in templating
 // # See: mmv1/compile/core.rb
 func commentBlock(text []string, lang string) string {