Add support for aliases like airflow-2 in Composer image version field (#5885) (#11422)

* Add support for Airflow major version aliases

This covers the new aliases for Airflow major version ('airflow-1',
'airflow-2') in image version field, and updates to the related
documentation.

* Use image version aliases in Composer tests

Instead of finding the latest available image version for the given
Composer and Airflow major versions, the new image version aliases are
used to let Composer resolve them on the server side:
'composer-1-airflow-1', 'composer-1-airflow-2', 'composer-2-airflow-2'.

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/5885.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+composer: Added support for `airflow-1` and `airflow-2` aliases in image version argument
+```

google/resource_composer_environment.go

@@ -17,7 +17,7 @@ import (
 const (
 	composerEnvironmentEnvVariablesRegexp          = "[a-zA-Z_][a-zA-Z0-9_]*."
 	composerEnvironmentReservedAirflowEnvVarRegexp = "AIRFLOW__[A-Z0-9_]+__[A-Z0-9_]+"
-	composerEnvironmentVersionRegexp               = `composer-(([0-9]+)(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-(([0-9]+\.[0-9]+)(\.[0-9]+)?)`
+	composerEnvironmentVersionRegexp               = `composer-(([0-9]+)(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-(([0-9]+)((\.[0-9]+)(\.[0-9]+)?)?)`
 )
 
 var composerEnvironmentReservedEnvVar = map[string]struct{}{
@@ -323,7 +323,7 @@ func resourceComposerEnvironment() *schema.Resource {
 										AtLeastOneOf:     composerSoftwareConfigKeys,
 										ValidateFunc:     validateRegexp(composerEnvironmentVersionRegexp),
 										DiffSuppressFunc: composerImageVersionDiffSuppress,
-										Description:      `The version of the software running in the environment. This encapsulates both the version of Cloud Composer functionality and the version of Apache Airflow. It must match the regular expression composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+\.[0-9]+(\.[0-9]+)?). The Cloud Composer portion of the image version is a full semantic version, or an alias in the form of major version number or 'latest'. The Apache Airflow portion of the image version is a full semantic version that points to one of the supported Apache Airflow versions, or an alias in the form of only major and minor versions specified. See documentation for more details and version list.`,
+										Description:      `The version of the software running in the environment. This encapsulates both the version of Cloud Composer functionality and the version of Apache Airflow. It must match the regular expression composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+(\.[0-9]+(\.[0-9]+)?)?). The Cloud Composer portion of the image version is a full semantic version, or an alias in the form of major version number or 'latest'. The Apache Airflow portion of the image version is a full semantic version that points to one of the supported Apache Airflow versions, or an alias in the form of only major or major.minor versions specified. See documentation for more details and version list.`,
 									},
 									"python_version": {
 										Type:         schema.TypeString,
@@ -1889,7 +1889,7 @@ func composerImageVersionDiffSuppress(_, old, new string, _ *schema.ResourceData
 	versionRe := regexp.MustCompile(composerEnvironmentVersionRegexp)
 	oldVersions := versionRe.FindStringSubmatch(old)
 	newVersions := versionRe.FindStringSubmatch(new)
-	if oldVersions == nil || len(oldVersions) < 8 {
+	if oldVersions == nil || len(oldVersions) < 10 {
 		// Somehow one of the versions didn't match the regexp or didn't
 		// have values in the capturing groups. In that case, fall back to
 		// an equality check.
@@ -1898,7 +1898,7 @@ func composerImageVersionDiffSuppress(_, old, new string, _ *schema.ResourceData
 		}
 		return old == new
 	}
-	if newVersions == nil || len(newVersions) < 8 {
+	if newVersions == nil || len(newVersions) < 10 {
 		// Somehow one of the versions didn't match the regexp or didn't
 		// have values in the capturing groups. In that case, fall back to
 		// an equality check.
@@ -1909,11 +1909,23 @@ func composerImageVersionDiffSuppress(_, old, new string, _ *schema.ResourceData
 	}
 
 	oldAirflow := oldVersions[5]
-	oldAirflowMajorMinor := oldVersions[6]
+	oldAirflowMajor := oldVersions[6]
+	oldAirflowMajorMinor := oldVersions[6] + oldVersions[8]
 	newAirflow := newVersions[5]
-	newAirflowMajorMinor := newVersions[6]
+	newAirflowMajor := newVersions[6]
+	newAirflowMajorMinor := newVersions[6] + newVersions[8]
 	// Check Airflow versions.
-	if oldAirflow == oldAirflowMajorMinor || newAirflow == newAirflowMajorMinor {
+	if oldAirflow == oldAirflowMajor || newAirflow == newAirflowMajor {
+		// If one of the Airflow versions specifies only major version
+		// (like 1), we can only compare major versions.
+		eq, err := versionsEqual(oldAirflowMajor, newAirflowMajor)
+		if err != nil {
+			log.Printf("[WARN] Could not parse airflow version, %s", err)
+		}
+		if !eq {
+			return false
+		}
+	} else if oldAirflow == oldAirflowMajorMinor || newAirflow == newAirflowMajorMinor {
 		// If one of the Airflow versions specifies only major and minor version
 		// (like 1.10), we can only compare major and minor versions.
 		eq, err := versionsEqual(oldAirflowMajorMinor, newAirflowMajorMinor)

google/resource_composer_environment_test.go

@@ -38,11 +38,13 @@ func TestComposerImageVersionDiffSuppress(t *testing.T) {
 		{"preview matches", "composer-1.17.0-preview.0-airflow-2.0.1", "composer-1.17.0-preview.0-airflow-2.0.1", true},
 		{"old latest", "composer-latest-airflow-1.10.0", "composer-1.4.1-airflow-1.10.0", true},
 		{"new latest", "composer-1.4.1-airflow-1.10.0", "composer-latest-airflow-1.10.0", true},
-		{"composer alias equivalent", "composer-1.4.0-airflow-1.10.0", "composer-1-airflow-1.10", true},
-		{"composer alias different", "composer-1.4.0-airflow-2.1.4", "composer-2-airflow-2.2", false},
+		{"composer major alias equivalent", "composer-1.4.0-airflow-1.10.0", "composer-1-airflow-1.10", true},
+		{"composer major alias different", "composer-1.4.0-airflow-2.1.4", "composer-2-airflow-2.2", false},
 		{"composer different", "composer-1.4.0-airflow-1.10.0", "composer-1.4.1-airflow-1.10.0", false},
-		{"airflow alias equivalent", "composer-1.4.0-airflow-1.10.0", "composer-1.4.0-airflow-1.10", true},
-		{"airflow alias different", "composer-1.4.0-airflow-2.1.4", "composer-1.4.0-airflow-2.2", false},
+		{"airflow major alias equivalent", "composer-1.4.0-airflow-1.10.0", "composer-1.4.0-airflow-1", true},
+		{"airflow major alias different", "composer-1.4.0-airflow-1.10.0", "composer-1.4.0-airflow-2", false},
+		{"airflow major.minor alias equivalent", "composer-1.4.0-airflow-1.10.0", "composer-1.4.0-airflow-1.10", true},
+		{"airflow major.minor alias different", "composer-1.4.0-airflow-2.1.4", "composer-1.4.0-airflow-2.2", false},
 		{"airflow different", "composer-1.4.0-airflow-1.10.0", "composer-1.4.0-airflow-1.9.0", false},
 	}
 
@@ -1099,15 +1101,6 @@ resource "google_compute_subnetwork" "test" {
 
 func testAccComposerEnvironment_composerV2(envName, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "2"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "2"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-east1"
@@ -1122,7 +1115,7 @@ resource "google_composer_environment" "test" {
     	}
 
   		software_config {
-  		  image_version = local.matching_images[0]
+  		  image_version = "composer-2-airflow-2"
   		}
 
   		workloads_config {
@@ -1174,15 +1167,6 @@ resource "google_compute_subnetwork" "test" {
 
 func testAccComposerEnvironment_composerV2PrivateServiceConnect(envName, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "2"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "2"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-central1"
@@ -1194,7 +1178,7 @@ resource "google_composer_environment" "test" {
     	}
 
   		software_config {
-  		  image_version = local.matching_images[0]
+  		  image_version = "composer-2-airflow-2"
   		}
   		private_environment_config {
 				cloud_composer_connection_subnetwork		= google_compute_subnetwork.test.self_link
@@ -1221,16 +1205,6 @@ resource "google_compute_subnetwork" "test" {
 
 func testAccComposerEnvironment_update(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "1"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "1"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
-
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-central1"
@@ -1249,7 +1223,7 @@ resource "google_composer_environment" "test" {
 		}
 
 		software_config {
-			image_version = local.matching_images[0]
+			image_version = "composer-1-airflow-1"
 
 			airflow_config_overrides = {
 				core-load_example = "True"
@@ -1289,15 +1263,6 @@ resource "google_compute_subnetwork" "test" {
 
 func testAccComposerEnvironment_updateComposerV2(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "2"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "2"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-east1"
@@ -1312,7 +1277,7 @@ resource "google_composer_environment" "test" {
     	}
 
   		software_config {
-  		  image_version = local.matching_images[0]
+  		  image_version = "composer-2-airflow-2"
   		}
 
   		workloads_config {
@@ -1411,16 +1376,6 @@ resource "google_project_iam_member" "composer-worker" {
 
 func testAccComposerEnvironment_softwareCfg(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "1"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "1"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
-
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-central1"
@@ -1431,7 +1386,7 @@ resource "google_composer_environment" "test" {
 			zone       = "us-central1-a"
 		}
 		software_config {
-			image_version  = local.matching_images[0]
+			image_version  = "composer-1-airflow-1"
 			python_version = "3"
 		}
 	}
@@ -1490,17 +1445,6 @@ resource "google_compute_subnetwork" "test" {
 
 func testAccComposerEnvironment_airflow2SoftwareCfg(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
-data "google_composer_image_versions" "all" {
-}
-
-locals {
-	composer_version = "1"  # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
-	airflow_version = "2"   # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
-	reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
-	matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
-}
-
-
 resource "google_composer_environment" "test" {
 	name   = "%s"
 	region = "us-central1"
@@ -1511,7 +1455,7 @@ resource "google_composer_environment" "test" {
 			zone       = "us-central1-a"
 		}
 		software_config {
-			image_version  = local.matching_images[0]
+			image_version  = "composer-1-airflow-2"
 			scheduler_count = 2
 		}
 	}

website/docs/r/composer_environment.html.markdown

@@ -60,7 +60,7 @@ resource "google_composer_environment" "test" {
  
  config {
     software_config {
-      image_version = "composer-2-airflow-2.2.3"
+      image_version = "composer-2-airflow-2"
     }
   }
 }
@@ -157,7 +157,7 @@ resource "google_composer_environment" "test" {
   config {
  
     software_config {
-      image_version = "composer-2-airflow-2.2.3"
+      image_version = "composer-2-airflow-2"
     }
  
     workloads_config {
@@ -451,11 +451,11 @@ The following arguments are supported:
 
   The version of the software running in the environment. This encapsulates both the version of Cloud Composer
   functionality and the version of Apache Airflow. It must match the regular expression
-  `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+\.[0-9]+(\.[0-9]+)?)`.
+  `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+(\.[0-9]+(\.[0-9]+)?)?)`.
   The Cloud Composer portion of the image version is a full semantic version, or an alias in the form of major
   version number or 'latest'.
   The Apache Airflow portion of the image version is a full semantic version that points to one of the
-  supported Apache Airflow versions, or an alias in the form of only major and minor versions specified.
+  supported Apache Airflow versions, or an alias in the form of only major or major.minor versions specified.
   For more information about Cloud Composer images, see
   [Cloud Composer version list](https://cloud.google.com/composer/docs/concepts/versioning/composer-versions).
 
@@ -765,11 +765,11 @@ The `software_config` block supports:
 
   The version of the software running in the environment. This encapsulates both the version of Cloud Composer
   functionality and the version of Apache Airflow. It must match the regular expression
-  `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+\.[0-9]+(\.[0-9]+)?)`.
+  `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+(\.[0-9]+(\.[0-9]+)?)?)`.
   The Cloud Composer portion of the image version is a full semantic version, or an alias in the form of major
   version number or 'latest'.
   The Apache Airflow portion of the image version is a full semantic version that points to one of the
-  supported Apache Airflow versions, or an alias in the form of only major and minor versions specified.
+  supported Apache Airflow versions, or an alias in the form of only major or major.minor versions specified.
   **Important**: You can only upgrade in-place between minor or patch versions of Cloud Composer or Apache
   Airflow. For example, you can upgrade your environment from `composer-1.16.x` to `composer-1.17.x`, or from
   `airflow-2.1.x` to `airflow-2.2.x`. You cannot upgrade between major Cloud Composer or Apache Airflow