TeamCity: Add project for testing the provider functions feature branch (#10088)

* Add ability to use non-default versions of Terraform in TeamCity builds

* Add function to enable making build configs for single packages at a time

* Add new sub project that contains 2 builds for testing provider functions

the 2 builds:
1) only pulls code from the feature branch on the downstream hashicorp/terraform-provider-google repo
2) only pulls code from the feature branch on the downstream hashicorp/terraform-provider-google-beta repo

These builds both use an alpha release of TF 1.8.0

* Add builds for testing auto generated branches in the MM upstream repos

These re-use existing VCR Roots.

* Make the builds that test the `FEATURE-BRANCH-provider-functions branches in the downstream repos run every night at the default time

* Fix defect in 'Download Terraform' build step definition

* Update build step to solve bug

* Update build_configuration_per_package.kt

Author: SarahFrench

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt

@@ -15,6 +15,8 @@ import jetbrains.buildServer.configs.kotlin.sharedResources
 import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
 import replaceCharsId
 
+// BuildConfigurationsForPackages accepts a map containing details of multiple packages in a provider and returns a list of build configurations for them all.
+// Intended to be used in projects where we're testing all packages, e.g. the nightly test projects
 fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): List<BuildType> {
     val list = ArrayList<BuildType>()
 
@@ -31,6 +33,13 @@ fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, p
     return list
 }
 
+// BuildConfigurationForSinglePackage accepts details of a single package in a provider and returns a build configuration for it
+// Intended to be used in short-lived projects where we're testing specific packages, e.g. feature branch testing
+fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): BuildType{
+    val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName)
+    return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables)
+}
+
 class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String) {
 
     // buildConfiguration returns a BuildType for a service package
@@ -102,4 +111,4 @@ class PackageDetails(private val packageName: String, private val displayName: S
         var id = "%s_%s_PACKAGE_%s".format(this.parentProjectName, this.providerName, this.packageName)
         return replaceCharsId(id)
     }
-}
+}

mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt

@@ -252,12 +252,20 @@ fun ParametrizedWithType.readOnlySettings() {
 }
 
 // ParametrizedWithType.terraformCoreBinaryTesting sets environment variables that control what Terraform version is downloaded
-// and ensures the testing framework uses that downloaded version
-fun ParametrizedWithType.terraformCoreBinaryTesting() {
-    text("env.TERRAFORM_CORE_VERSION", DefaultTerraformCoreVersion, "The version of Terraform Core which should be used for testing")
+// and ensures the testing framework uses that downloaded version. The default Terraform core version is used if no argument is supplied.
+fun ParametrizedWithType.terraformCoreBinaryTesting(tfVersion: String = DefaultTerraformCoreVersion) {
+    text("env.TERRAFORM_CORE_VERSION", tfVersion, "The version of Terraform Core which should be used for testing")
     hiddenVariable("env.TF_ACC_TERRAFORM_PATH", "%system.teamcity.build.checkoutDir%/tools/terraform", "The path where the Terraform Binary is located. Used by the testing framework.")
 }
 
+// BuildType.overrideTerraformCoreVersion is used to override the value of TERRAFORM_CORE_VERSION in special cases where we're testing new features
+// that rely on a specific version of Terraform we might not want to be used for all our tests in TeamCity.
+fun BuildType.overrideTerraformCoreVersion(tfVersion: String){
+    params {
+        terraformCoreBinaryTesting(tfVersion)
+    }
+}
+
 fun ParametrizedWithType.terraformShouldPanicForSchemaErrors() {
     hiddenVariable("env.TF_SCHEMA_PANIC_ON_ERROR", "1", "Panic if unknown/unmatched fields are set into the state")
 }

mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt

@@ -64,14 +64,15 @@ fun BuildSteps.downloadTerraformBinary() {
     // https://releases.hashicorp.com/terraform/0.12.28/terraform_0.12.28_linux_amd64.zip
     val terraformUrl = "https://releases.hashicorp.com/terraform/%env.TERRAFORM_CORE_VERSION%/terraform_%env.TERRAFORM_CORE_VERSION%_linux_amd64.zip"
     step(ScriptBuildStep {
-        name = "Download Terraform version %s".format(DefaultTerraformCoreVersion)
+        name = "Download Terraform"
         scriptContent = """
         #!/bin/bash
+        echo "Downloading Terraform version %env.TERRAFORM_CORE_VERSION%"
         mkdir -p tools
-        wget -O tf.zip %s
+        wget -O tf.zip $terraformUrl
         unzip tf.zip
         mv terraform tools/
-        """.format(terraformUrl).trimIndent()
+        """.trimIndent()
     })
 }
 

mmv1/third_party/terraform/.teamcity/components/inputs/packages.kt

@@ -13,6 +13,11 @@ var PackagesListGa = mapOf(
         "displayName" to "Environment Variables",
         "path" to "./google/envvar"
     ),
+    "functions" to mapOf(
+        "name" to "functions",
+        "displayName" to "Provider-Defined Functions",
+        "path" to "./google/functions"
+    ),
     "fwmodels" to mapOf(
         "name" to "fwmodels",
         "displayName" to "Framework Models",
@@ -64,6 +69,11 @@ var PackagesListBeta = mapOf(
         "displayName" to "Environment Variables",
         "path" to "./google-beta/envvar"
     ),
+    "functions" to mapOf(
+        "name" to "functions",
+        "displayName" to "Provider-Defined Functions",
+        "path" to "./google-beta/functions"
+    ),
     "fwmodels" to mapOf(
         "name" to "fwmodels",
         "displayName" to "Framework Models",

mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-provider-functions.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is controlled by MMv1, any changes made here will be overwritten
+
+package projects.feature_branches
+
+import ProviderNameBeta
+import ProviderNameGa
+import builds.*
+import generated.PackagesListBeta
+import generated.PackagesListGa
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
+import replaceCharsId
+import vcs_roots.ModularMagicianVCSRootBeta
+import vcs_roots.ModularMagicianVCSRootGa
+
+const val featureBranchProviderFunctionsName = "FEATURE-BRANCH-provider-functions"
+const val providerFunctionsTfCoreVersion = "1.8.0-alpha20240228"
+
+// VCS Roots specifically for pulling code from the feature branches in the downstream and upstream repos
+object HashicorpVCSRootGa_featureBranchProviderFunctions: GitVcsRoot({
+    name = "VCS root for the hashicorp/terraform-provider-${ProviderNameGa} repo @ refs/heads/${featureBranchProviderFunctionsName}"
+    url = "https://github.com/hashicorp/terraform-provider-${ProviderNameGa}"
+    branch = "refs/heads/${featureBranchProviderFunctionsName}"
+    branchSpec = "" // empty as we'll access no other branches
+})
+
+object HashicorpVCSRootBeta_featureBranchProviderFunctions: GitVcsRoot({
+    name = "VCS root for the hashicorp/terraform-provider-${ProviderNameBeta} repo @ refs/heads/${featureBranchProviderFunctionsName}"
+    url = "https://github.com/hashicorp/terraform-provider-${ProviderNameBeta}"
+    branch = "refs/heads/${featureBranchProviderFunctionsName}"
+    branchSpec = "" // empty as we'll access no other branches
+})
+
+fun featureBranchProviderFunctionSubProject(allConfig: AllContextParameters): Project {
+
+    val projectId = replaceCharsId(featureBranchProviderFunctionsName)
+
+    val packageName = "functions" // This project will contain only builds to test this single package
+    val sharedResourcesEmpty: List<String> = listOf() // No locking when testing functions
+    val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs
+    val trigger  = NightlyTriggerConfiguration() // Resued below for running tests against the downstream repos every night.
+
+    var parentId: String // To be overwritten when each build config is generated below.
+
+    // GA
+    val gaConfig = getGaAcceptanceTestConfig(allConfig)
+    // How to make only build configuration to the relevant package(s)
+    val functionPackageGa = PackagesListGa.getValue(packageName)
+
+    // Enable testing using hashicorp/terraform-provider-google
+    parentId = "${projectId}_HC_GA"
+    val buildConfigHashiCorpGa = BuildConfigurationForSinglePackage(packageName, functionPackageGa.getValue("path"), "Provider-Defined Functions (GA provider, HashiCorp downstream)", ProviderNameGa, parentId, HashicorpVCSRootGa_featureBranchProviderFunctions, sharedResourcesEmpty, gaConfig)
+    buildConfigHashiCorpGa.addTrigger(trigger)
+
+    // Enable testing using modular-magician/terraform-provider-google
+    parentId = "${projectId}_MM_GA"
+    val buildConfigModularMagicianGa = BuildConfigurationForSinglePackage(packageName, functionPackageGa.getValue("path"), "Provider-Defined Functions (GA provider, MM upstream)", ProviderNameGa, parentId, ModularMagicianVCSRootGa, sharedResourcesEmpty, vcrConfig)
+
+    // Beta
+    val betaConfig = getBetaAcceptanceTestConfig(allConfig)
+    val functionPackageBeta = PackagesListBeta.getValue("functions")
+
+    // Enable testing using hashicorp/terraform-provider-google-beta
+    parentId = "${projectId}_HC_BETA"
+    val buildConfigHashiCorpBeta = BuildConfigurationForSinglePackage(packageName, functionPackageBeta.getValue("path"), "Provider-Defined Functions (Beta provider, HashiCorp downstream)", ProviderNameBeta, parentId, HashicorpVCSRootBeta_featureBranchProviderFunctions, sharedResourcesEmpty, betaConfig)
+    buildConfigHashiCorpBeta.addTrigger(trigger)
+
+    // Enable testing using modular-magician/terraform-provider-google-beta
+    parentId = "${projectId}_MM_BETA"
+    val buildConfigModularMagicianBeta = BuildConfigurationForSinglePackage(packageName, functionPackageBeta.getValue("path"), "Provider-Defined Functions (Beta provider, MM upstream)", ProviderNameBeta, parentId, ModularMagicianVCSRootBeta, sharedResourcesEmpty, vcrConfig)
+
+    val allBuildConfigs = listOf(buildConfigHashiCorpGa, buildConfigModularMagicianGa, buildConfigHashiCorpBeta, buildConfigModularMagicianBeta)
+
+    // Make these builds use a 1.8.0-ish version of TF core
+    allBuildConfigs.forEach{ b ->
+        b.overrideTerraformCoreVersion(providerFunctionsTfCoreVersion)
+    }
+
+    return Project{
+        id(projectId)
+        name = featureBranchProviderFunctionsName
+        description = "Subproject for testing feature branch $featureBranchProviderFunctionsName"
+
+        // Register feature branch-specific VCS roots in the project
+        vcsRoot(HashicorpVCSRootGa_featureBranchProviderFunctions)
+        vcsRoot(HashicorpVCSRootBeta_featureBranchProviderFunctions)
+
+        // Register all build configs in the project
+        allBuildConfigs.forEach{ b ->
+            buildType(b)
+        }
+
+        params {
+            readOnlySettings()
+        }
+    }
+}

mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt

@@ -18,6 +18,7 @@ import generated.ServicesListBeta
 import generated.ServicesListGa
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.sharedResource
+import projects.feature_branches.featureBranchProviderFunctionSubProject
 
 // googleCloudRootProject returns a root project that contains a subprojects for the GA and Beta version of the
 // Google provider. There are also resources to help manage the test projects used for acceptance tests.
@@ -57,10 +58,14 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
             }
         }
 
+        // Projects required for nightly testing, testing MM upstreams, and sweepers
         subProject(googleSubProjectGa(allConfig))
         subProject(googleSubProjectBeta(allConfig))
         subProject(projectSweeperSubProject(allConfig))
 
+        // Feature branch-testing projects - these will be added and removed as needed
+        subProject(featureBranchProviderFunctionSubProject(allConfig))
+
         params {
             readOnlySettings()
         }