Skip to content

Commit 2ec420e

Browse files
modular-magicianmodular-magician
authored
Add Scheduled snapshots for CC (#7065) (#13504)
Co-authored-by: rlapin-pl <[email protected]> Fixes #13360 Signed-off-by: Modular Magician <[email protected]> Signed-off-by: Modular Magician <[email protected]>
1 parent eefc1a5 commit 2ec420e

File tree

4 files changed

+278
-0
lines changed

4 files changed

+278
-0
lines changed

‎.changelog/7065.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
composer: added `recovery_config` in `google_composer_environment` resource
3+
```

‎google/resource_composer_environment.go

+115
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ var (
5050
"config.0.node_count",
5151
"config.0.node_config",
5252
"config.0.software_config",
53+
"config.0.recovery_config",
5354
"config.0.private_environment_config",
5455
"config.0.web_server_network_access_control",
5556
"config.0.database_config",
@@ -61,6 +62,10 @@ var (
6162
"config.0.master_authorized_networks_config",
6263
}
6364

65+
recoveryConfigKeys = []string{
66+
"config.0.recovery_config.0.scheduled_snapshots_config",
67+
}
68+
6469
workloadsConfigKeys = []string{
6570
"config.0.workloads_config.0.scheduler",
6671
"config.0.workloads_config.0.web_server",
@@ -314,6 +319,48 @@ func resourceComposerEnvironment() *schema.Resource {
314319
},
315320
},
316321
},
322+
"recovery_config": {
323+
Type: schema.TypeList,
324+
Optional: true,
325+
AtLeastOneOf: composerConfigKeys,
326+
MaxItems: 1,
327+
Description: `The recovery configuration settings for the Cloud Composer environment`,
328+
Elem: &schema.Resource{
329+
Schema: map[string]*schema.Schema{
330+
"scheduled_snapshots_config": {
331+
Type: schema.TypeList,
332+
Optional: true,
333+
AtLeastOneOf: recoveryConfigKeys,
334+
Description: `The configuration settings for scheduled snapshots.`,
335+
MaxItems: 1,
336+
Elem: &schema.Resource{
337+
Schema: map[string]*schema.Schema{
338+
"enabled": {
339+
Type: schema.TypeBool,
340+
Required: true,
341+
Description: `When enabled, Cloud Composer periodically saves snapshots of your environment to a Cloud Storage bucket.`,
342+
},
343+
"snapshot_location": {
344+
Type: schema.TypeString,
345+
Optional: true,
346+
Description: `the URI of a bucket folder where to save the snapshot.`,
347+
},
348+
"snapshot_creation_schedule": {
349+
Type: schema.TypeString,
350+
Optional: true,
351+
Description: `Snapshot schedule, in the unix-cron format.`,
352+
},
353+
"time_zone": {
354+
Type: schema.TypeString,
355+
Optional: true,
356+
Description: `A time zone for the schedule. This value is a time offset and does not take into account daylight saving time changes. Valid values are from UTC-12 to UTC+12. Examples: UTC, UTC-01, UTC+03.`,
357+
},
358+
},
359+
},
360+
},
361+
},
362+
},
363+
},
317364
"software_config": {
318365
Type: schema.TypeList,
319366
Optional: true,
@@ -996,6 +1043,18 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
9961043
return err
9971044
}
9981045
}
1046+
1047+
if d.HasChange("config.0.recovery_config.0.scheduled_snapshots_config") {
1048+
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
1049+
if config != nil {
1050+
patchObj.Config.RecoveryConfig = config.RecoveryConfig
1051+
}
1052+
err = resourceComposerEnvironmentPatchField("config.RecoveryConfig.ScheduledSnapshotsConfig", userAgent, patchObj, d, tfConfig)
1053+
if err != nil {
1054+
return err
1055+
}
1056+
}
1057+
9991058
if d.HasChange("config.0.environment_size") {
10001059
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
10011060
if config != nil {
@@ -1140,6 +1199,7 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
11401199
transformed["encryption_config"] = flattenComposerEnvironmentConfigEncryptionConfig(envCfg.EncryptionConfig)
11411200
transformed["maintenance_window"] = flattenComposerEnvironmentConfigMaintenanceWindow(envCfg.MaintenanceWindow)
11421201
transformed["workloads_config"] = flattenComposerEnvironmentConfigWorkloadsConfig(envCfg.WorkloadsConfig)
1202+
transformed["recovery_config"] = flattenComposerEnvironmentConfigRecoveryConfig(envCfg.RecoveryConfig)
11431203
transformed["environment_size"] = envCfg.EnvironmentSize
11441204
transformed["master_authorized_networks_config"] = flattenComposerEnvironmentConfigMasterAuthorizedNetworksConfig(envCfg.MasterAuthorizedNetworksConfig)
11451205
return []interface{}{transformed}
@@ -1199,6 +1259,30 @@ func flattenComposerEnvironmentConfigEncryptionConfig(encryptionCfg *composer.En
11991259
return []interface{}{transformed}
12001260
}
12011261

1262+
func flattenComposerEnvironmentConfigRecoveryConfig(recoveryCfg *composer.RecoveryConfig) interface{} {
1263+
if recoveryCfg == nil {
1264+
return nil
1265+
}
1266+
1267+
transformed := make(map[string]interface{})
1268+
transformedScheduledSnapshotsConfig := make(map[string]interface{})
1269+
1270+
scheduledSnapshotsConfig := recoveryCfg.ScheduledSnapshotsConfig
1271+
1272+
if scheduledSnapshotsConfig == nil {
1273+
transformedScheduledSnapshotsConfig = nil
1274+
} else {
1275+
transformedScheduledSnapshotsConfig["enabled"] = scheduledSnapshotsConfig.Enabled
1276+
transformedScheduledSnapshotsConfig["snapshot_location"] = scheduledSnapshotsConfig.SnapshotLocation
1277+
transformedScheduledSnapshotsConfig["time_zone"] = scheduledSnapshotsConfig.TimeZone
1278+
transformedScheduledSnapshotsConfig["snapshot_creation_schedule"] = scheduledSnapshotsConfig.SnapshotCreationSchedule
1279+
}
1280+
1281+
transformed["scheduled_snapshots_config"] = []interface{}{transformedScheduledSnapshotsConfig}
1282+
1283+
return []interface{}{transformed}
1284+
}
1285+
12021286
func flattenComposerEnvironmentConfigMaintenanceWindow(maintenanceWindow *composer.MaintenanceWindow) interface{} {
12031287
if maintenanceWindow == nil {
12041288
return nil
@@ -1437,6 +1521,13 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
14371521
return nil, err
14381522
}
14391523
transformed.MasterAuthorizedNetworksConfig = transformedMasterAuthorizedNetworksConfig
1524+
1525+
transformedRecoveryConfig, err := expandComposerEnvironmentConfigRecoveryConfig(original["recovery_config"], d, config)
1526+
if err != nil {
1527+
return nil, err
1528+
}
1529+
transformed.RecoveryConfig = transformedRecoveryConfig
1530+
14401531
return transformed, nil
14411532
}
14421533

@@ -1617,6 +1708,30 @@ func expandComposerEnvironmentConfigWorkloadsConfig(v interface{}, d *schema.Res
16171708
return transformed, nil
16181709
}
16191710

1711+
func expandComposerEnvironmentConfigRecoveryConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.RecoveryConfig, error) {
1712+
l := v.([]interface{})
1713+
if len(l) == 0 {
1714+
return nil, nil
1715+
}
1716+
raw := l[0]
1717+
original := raw.(map[string]interface{})
1718+
transformed := &composer.RecoveryConfig{}
1719+
1720+
if v, ok := original["scheduled_snapshots_config"]; ok {
1721+
if len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
1722+
transformedScheduledSnapshotsConfig := &composer.ScheduledSnapshotsConfig{}
1723+
originalScheduledSnapshotsConfigRaw := v.([]interface{})[0].(map[string]interface{})
1724+
transformedScheduledSnapshotsConfig.Enabled = originalScheduledSnapshotsConfigRaw["enabled"].(bool)
1725+
transformedScheduledSnapshotsConfig.SnapshotLocation = originalScheduledSnapshotsConfigRaw["snapshot_location"].(string)
1726+
transformedScheduledSnapshotsConfig.TimeZone = originalScheduledSnapshotsConfigRaw["time_zone"].(string)
1727+
transformedScheduledSnapshotsConfig.SnapshotCreationSchedule = originalScheduledSnapshotsConfigRaw["snapshot_creation_schedule"].(string)
1728+
transformed.ScheduledSnapshotsConfig = transformedScheduledSnapshotsConfig
1729+
}
1730+
}
1731+
1732+
return transformed, nil
1733+
}
1734+
16201735
func expandComposerEnvironmentConfigEnvironmentSize(v interface{}, d *schema.ResourceData, config *Config) (string, error) {
16211736
if v == nil {
16221737
return "", nil

‎google/resource_composer_environment_test.go

+132
Original file line numberDiff line numberDiff line change
@@ -744,6 +744,46 @@ func TestAccComposerEnvironment_withNodeConfig(t *testing.T) {
744744
})
745745
}
746746

747+
func TestAccComposerEnvironmentAirflow2_withRecoveryConfig(t *testing.T) {
748+
t.Parallel()
749+
envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
750+
network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, randInt(t))
751+
subnetwork := network + "-1"
752+
753+
vcrTest(t, resource.TestCase{
754+
PreCheck: func() { testAccPreCheck(t) },
755+
Providers: testAccProviders,
756+
CheckDestroy: testAccComposerEnvironmentDestroyProducer(t),
757+
Steps: []resource.TestStep{
758+
{
759+
Config: testAccComposerEnvironment_airflow2RecoveryCfg(envName, network, subnetwork),
760+
},
761+
{
762+
ResourceName: "google_composer_environment.test",
763+
ImportState: true,
764+
ImportStateVerify: true,
765+
},
766+
{
767+
Config: testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(envName, network, subnetwork),
768+
},
769+
{
770+
ResourceName: "google_composer_environment.test",
771+
ImportState: true,
772+
ImportStateVerify: true,
773+
},
774+
// This is a terrible clean-up step in order to get destroy to succeed,
775+
// due to dangling firewall rules left by the Composer Environment blocking network deletion.
776+
// TODO: Remove this check if firewall rules bug gets fixed by Composer.
777+
{
778+
PlanOnly: true,
779+
ExpectNonEmptyPlan: false,
780+
Config: testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(envName, network, subnetwork),
781+
Check: testAccCheckClearComposerEnvironmentFirewalls(t, network),
782+
},
783+
},
784+
})
785+
}
786+
747787
func TestAccComposerEnvironment_withSoftwareConfig(t *testing.T) {
748788
t.Parallel()
749789
envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
@@ -1792,6 +1832,98 @@ resource "google_project_iam_member" "composer-worker" {
17921832
`, environment, network, subnetwork, serviceAccount)
17931833
}
17941834

1835+
func testAccComposerEnvironment_airflow2RecoveryCfg(name, network, subnetwork string) string {
1836+
return fmt.Sprintf(`
1837+
resource "google_composer_environment" "test" {
1838+
name = "%s"
1839+
region = "us-central1"
1840+
1841+
config {
1842+
node_config {
1843+
network = google_compute_network.test.self_link
1844+
subnetwork = google_compute_subnetwork.test.self_link
1845+
ip_allocation_policy {
1846+
cluster_ipv4_cidr_block = "10.0.0.0/16"
1847+
}
1848+
}
1849+
1850+
software_config {
1851+
image_version = "composer-2-airflow-2"
1852+
}
1853+
1854+
recovery_config {
1855+
scheduled_snapshots_config {
1856+
enabled = true
1857+
snapshot_location = "gs://example-bucket/environment_snapshots"
1858+
snapshot_creation_schedule = "0 4 * * *"
1859+
time_zone = "UTC+01"
1860+
}
1861+
}
1862+
}
1863+
1864+
}
1865+
1866+
resource "google_compute_network" "test" {
1867+
name = "%s"
1868+
auto_create_subnetworks = false
1869+
}
1870+
1871+
resource "google_compute_subnetwork" "test" {
1872+
name = "%s"
1873+
ip_cidr_range = "10.2.0.0/16"
1874+
region = "us-central1"
1875+
network = google_compute_network.test.self_link
1876+
private_ip_google_access = true
1877+
}
1878+
`, name, network, subnetwork)
1879+
}
1880+
1881+
func testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(name, network, subnetwork string) string {
1882+
return fmt.Sprintf(`
1883+
resource "google_composer_environment" "test" {
1884+
name = "%s"
1885+
region = "us-central1"
1886+
1887+
config {
1888+
node_config {
1889+
network = google_compute_network.test.self_link
1890+
subnetwork = google_compute_subnetwork.test.self_link
1891+
ip_allocation_policy {
1892+
cluster_ipv4_cidr_block = "10.0.0.0/16"
1893+
}
1894+
}
1895+
1896+
software_config {
1897+
image_version = "composer-2-airflow-2"
1898+
}
1899+
1900+
recovery_config {
1901+
scheduled_snapshots_config {
1902+
enabled = true
1903+
snapshot_location = "gs://example-bucket/environment_snapshots2"
1904+
snapshot_creation_schedule = "1 2 * * *"
1905+
time_zone = "UTC+02"
1906+
}
1907+
}
1908+
}
1909+
1910+
}
1911+
1912+
resource "google_compute_network" "test" {
1913+
name = "%s"
1914+
auto_create_subnetworks = false
1915+
}
1916+
1917+
resource "google_compute_subnetwork" "test" {
1918+
name = "%s"
1919+
ip_cidr_range = "10.2.0.0/16"
1920+
region = "us-central1"
1921+
network = google_compute_network.test.self_link
1922+
private_ip_google_access = true
1923+
}
1924+
`, name, network, subnetwork)
1925+
}
1926+
17951927
func testAccComposerEnvironment_softwareCfg(name, network, subnetwork string) string {
17961928
return fmt.Sprintf(`
17971929
resource "google_composer_environment" "test" {

‎website/docs/r/composer_environment.html.markdown

+28
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,10 @@ The following arguments are supported:
281281
(Optional)
282282
The configuration used for the Kubernetes Engine cluster. Structure is [documented below](#nested_node_config).
283283

284+
* `recovery_config` -
285+
(Optional, Cloud Composer 2 only)
286+
The configuration settings for recovery. Structure is [documented below](#nested_recovery_config).
287+
284288
* `software_config` -
285289
(Optional)
286290
The configuration settings for software inside the environment. Structure is [documented below](#nested_software_config).
@@ -877,6 +881,30 @@ The `ip_allocation_policy` block supports:
877881
The only allowed values for 'FREQ' field are 'FREQ=DAILY' and 'FREQ=WEEKLY;BYDAY=...'.
878882
Example values: 'FREQ=WEEKLY;BYDAY=TU,WE', 'FREQ=DAILY'.
879883

884+
The `recovery_config` block supports:
885+
886+
* `scheduled_snapshots_config` -
887+
(Optional)
888+
The recovery configuration settings for the Cloud Composer environment.
889+
890+
The `scheduled_snapshots_config` block supports:
891+
892+
* `enabled` -
893+
(Optional)
894+
When enabled, Cloud Composer periodically saves snapshots of your environment to a Cloud Storage bucket.
895+
896+
* `snapshot_location` -
897+
(Optional)
898+
The URI of a bucket folder where to save the snapshot.
899+
900+
* `snapshot_creation_schedule` -
901+
(Optional)
902+
Snapshot schedule, in the unix-cron format.
903+
904+
* `time_zone` -
905+
(Optional)
906+
A time zone for the schedule. This value is a time offset and does not take into account daylight saving time changes. Valid values are from UTC-12 to UTC+12. Examples: UTC, UTC-01, UTC+03.
907+
880908
The `workloads_config` block supports:
881909

882910
* `scheduler` -

0 commit comments

Comments
 (0)