Skip to content

Commit 4b97647

Browse files
modular-magicianmodular-magician
authored
Add Cloud Composer Support for Logs in Cloud Logging Only feature (#9821) (#17050)
* Adding storage mode field * Fixing * Adding update and resolving PR comments * Fixing test * Resolving Comments * Fixing typo * Fixing documentation * Adding md * Fix indentation issues and typos --------- [upstream:d5045d4c99af96c042608b1ddf1de51ef7af9653] Signed-off-by: Modular Magician <[email protected]>
1 parent 35be711 commit 4b97647

File tree

4 files changed

+143
-1
lines changed

4 files changed

+143
-1
lines changed

‎.changelog/9821.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
composer: added `data_retention_config.task_logs_retention_config.storage_mode` field on `google_composer_environment`
3+
```

‎google/services/composer/resource_composer_environment.go

+111-1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ var (
6868
"config.0.environment_size",
6969
"config.0.master_authorized_networks_config",
7070
"config.0.resilience_mode",
71+
"config.0.data_retention_config",
7172
}
7273

7374
recoveryConfigKeys = []string{
@@ -623,6 +624,34 @@ func ResourceComposerEnvironment() *schema.Resource {
623624
},
624625
},
625626
},
627+
628+
"data_retention_config": {
629+
Type: schema.TypeList,
630+
Optional: true,
631+
Computed: true,
632+
AtLeastOneOf: composerConfigKeys,
633+
MaxItems: 1,
634+
Description: `The configuration setting for Airflow data retention mechanism. This field is supported for Cloud Composer environments in versions composer-2.0.32-airflow-2.1.4. or newer`,
635+
Elem: &schema.Resource{
636+
Schema: map[string]*schema.Schema{
637+
"task_logs_retention_config": {
638+
Type: schema.TypeList,
639+
Description: `Optional. The configuration setting for Task Logs.`,
640+
Required: true,
641+
Elem: &schema.Resource{
642+
Schema: map[string]*schema.Schema{
643+
"storage_mode": {
644+
Type: schema.TypeString,
645+
Optional: true,
646+
ValidateFunc: validation.StringInSlice([]string{"CLOUD_LOGGING_ONLY", "CLOUD_LOGGING_AND_CLOUD_STORAGE"}, false),
647+
Description: `Whether logs in cloud logging only is enabled or not. This field is supported for Cloud Composer environments in versions composer-2.0.32-airflow-2.1.4 and newer.`,
648+
},
649+
},
650+
},
651+
},
652+
},
653+
},
654+
},
626655
"workloads_config": {
627656
Type: schema.TypeList,
628657
Optional: true,
@@ -1178,7 +1207,22 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
11781207
return err
11791208
}
11801209
}
1181-
1210+
if d.HasChange("config.0.data_retention_config.0.task_logs_retention_config.0.storage_mode") {
1211+
patchObj := &composer.Environment{
1212+
Config: &composer.EnvironmentConfig{
1213+
DataRetentionConfig: &composer.DataRetentionConfig{
1214+
TaskLogsRetentionConfig: &composer.TaskLogsRetentionConfig{},
1215+
},
1216+
},
1217+
}
1218+
if config != nil && config.DataRetentionConfig != nil && config.DataRetentionConfig.TaskLogsRetentionConfig != nil {
1219+
patchObj.Config.DataRetentionConfig.TaskLogsRetentionConfig.StorageMode = config.DataRetentionConfig.TaskLogsRetentionConfig.StorageMode
1220+
}
1221+
err = resourceComposerEnvironmentPatchField("config.DataRetentionConfig.TaskLogsRetentionConfig.StorageMode", userAgent, patchObj, d, tfConfig)
1222+
if err != nil {
1223+
return err
1224+
}
1225+
}
11821226
if d.HasChange("config.0.recovery_config.0.scheduled_snapshots_config") {
11831227
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
11841228
if config != nil {
@@ -1356,6 +1400,7 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
13561400
transformed["web_server_config"] = flattenComposerEnvironmentConfigWebServerConfig(envCfg.WebServerConfig)
13571401
transformed["encryption_config"] = flattenComposerEnvironmentConfigEncryptionConfig(envCfg.EncryptionConfig)
13581402
transformed["maintenance_window"] = flattenComposerEnvironmentConfigMaintenanceWindow(envCfg.MaintenanceWindow)
1403+
transformed["data_retention_config"] = flattenComposerEnvironmentConfigDataRetentionConfig(envCfg.DataRetentionConfig)
13591404
transformed["workloads_config"] = flattenComposerEnvironmentConfigWorkloadsConfig(envCfg.WorkloadsConfig)
13601405
transformed["recovery_config"] = flattenComposerEnvironmentConfigRecoveryConfig(envCfg.RecoveryConfig)
13611406
transformed["environment_size"] = envCfg.EnvironmentSize
@@ -1460,6 +1505,28 @@ func flattenComposerEnvironmentConfigMaintenanceWindow(maintenanceWindow *compos
14601505
return []interface{}{transformed}
14611506
}
14621507

1508+
func flattenComposerEnvironmentConfigDataRetentionConfig(dataRetentionConfig *composer.DataRetentionConfig) interface{} {
1509+
if dataRetentionConfig == nil {
1510+
return nil
1511+
}
1512+
1513+
transformed := make(map[string]interface{})
1514+
transformed["task_logs_retention_config"] = flattenComposerEnvironmentConfigDataRetentionConfigTaskLogsRetentionConfig(dataRetentionConfig.TaskLogsRetentionConfig)
1515+
1516+
return []interface{}{transformed}
1517+
}
1518+
1519+
func flattenComposerEnvironmentConfigDataRetentionConfigTaskLogsRetentionConfig(taskLogsRetentionConfig *composer.TaskLogsRetentionConfig) interface{} {
1520+
if taskLogsRetentionConfig == nil {
1521+
return nil
1522+
}
1523+
1524+
transformed := make(map[string]interface{})
1525+
transformed["storage_mode"] = taskLogsRetentionConfig.StorageMode
1526+
1527+
return []interface{}{transformed}
1528+
}
1529+
14631530
func flattenComposerEnvironmentConfigWorkloadsConfig(workloadsConfig *composer.WorkloadsConfig) interface{} {
14641531
if workloadsConfig == nil {
14651532
return nil
@@ -1687,6 +1754,13 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
16871754
return nil, err
16881755
}
16891756
transformed.MaintenanceWindow = transformedMaintenanceWindow
1757+
1758+
transformedDataRetentionConfig, err := expandComposerEnvironmentConfigDataRetentionConfig(original["data_retention_config"], d, config)
1759+
if err != nil {
1760+
return nil, err
1761+
}
1762+
transformed.DataRetentionConfig = transformedDataRetentionConfig
1763+
16901764
transformedWorkloadsConfig, err := expandComposerEnvironmentConfigWorkloadsConfig(original["workloads_config"], d, config)
16911765
if err != nil {
16921766
return nil, err
@@ -1854,6 +1928,42 @@ func expandComposerEnvironmentConfigMaintenanceWindow(v interface{}, d *schema.R
18541928
return transformed, nil
18551929
}
18561930

1931+
func expandComposerEnvironmentConfigDataRetentionConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (*composer.DataRetentionConfig, error) {
1932+
l := v.([]interface{})
1933+
if len(l) == 0 {
1934+
return nil, nil
1935+
}
1936+
raw := l[0]
1937+
original := raw.(map[string]interface{})
1938+
transformed := &composer.DataRetentionConfig{}
1939+
1940+
if taskLogsRetentionConfig, ok := original["task_logs_retention_config"]; ok {
1941+
transformedTaskLogsRetentionConfig, err := expandComposerEnvironmentConfigDataRetentionConfigTaskLogsRetentionConfig(taskLogsRetentionConfig, d, config)
1942+
if err != nil {
1943+
return nil, err
1944+
}
1945+
transformed.TaskLogsRetentionConfig = transformedTaskLogsRetentionConfig
1946+
}
1947+
1948+
return transformed, nil
1949+
}
1950+
1951+
func expandComposerEnvironmentConfigDataRetentionConfigTaskLogsRetentionConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (*composer.TaskLogsRetentionConfig, error) {
1952+
l := v.([]interface{})
1953+
if len(l) == 0 {
1954+
return nil, nil
1955+
}
1956+
raw := l[0]
1957+
original := raw.(map[string]interface{})
1958+
transformed := &composer.TaskLogsRetentionConfig{}
1959+
1960+
if v, ok := original["storage_mode"]; ok {
1961+
transformed.StorageMode = v.(string)
1962+
}
1963+
1964+
return transformed, nil
1965+
}
1966+
18571967
func expandComposerEnvironmentConfigWorkloadsConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (*composer.WorkloadsConfig, error) {
18581968
l := v.([]interface{})
18591969
if len(l) == 0 {

‎google/services/composer/resource_composer_environment_test.go

+10
Original file line numberDiff line numberDiff line change
@@ -1753,6 +1753,11 @@ resource "google_composer_environment" "test" {
17531753
zone = "us-east1-c"
17541754
}
17551755
environment_size = "ENVIRONMENT_SIZE_MEDIUM"
1756+
data_retention_config {
1757+
task_logs_retention_config {
1758+
storage_mode = "CLOUD_LOGGING_ONLY"
1759+
}
1760+
}
17561761
private_environment_config {
17571762
enable_private_endpoint = true
17581763
cloud_composer_network_ipv4_cidr_block = "10.3.192.0/24"
@@ -2131,6 +2136,11 @@ resource "google_composer_environment" "test" {
21312136
}
21322137
}
21332138
environment_size = "ENVIRONMENT_SIZE_LARGE"
2139+
data_retention_config {
2140+
task_logs_retention_config {
2141+
storage_mode = "CLOUD_LOGGING_AND_CLOUD_STORAGE"
2142+
}
2143+
}
21342144
private_environment_config {
21352145
enable_private_endpoint = true
21362146
cloud_composer_network_ipv4_cidr_block = "10.3.192.0/24"

‎website/docs/r/composer_environment.html.markdown

+19
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,25 @@ The `config` block supports:
711711
Google Compute Engine Public IPs and Google Prod IPs. Structure is
712712
documented below.
713713

714+
* `data_retention_config` -
715+
(Optional, Cloud Composer 2.0.23 or newer only)
716+
Configuration setting for airflow data rentention mechanism. Structure is
717+
[documented below](#nested_data_retention_config).
718+
719+
<a name="nested_data_retention_config"></a>The `data_retention_config` block supports:
720+
* `task_logs_retention_config` -
721+
(Optional)
722+
The configuration setting for Task Logs. Structure is
723+
[documented below](#nested_task_logs_retention_config).
724+
725+
<a name="nested_task_logs_retention_config"></a>The `task_logs_retention_config` block supports:
726+
* `storage_mode` -
727+
(Optional)
728+
The mode of storage for Airflow workers task logs. Values for storage mode are
729+
`CLOUD_LOGGING_ONLY` to only store logs in cloud logging and
730+
`CLOUD_LOGGING_AND_CLOUD_STORAGE` to store logs in cloud logging and cloud storage.
731+
732+
714733
The `storage_config` block supports:
715734

716735
* `bucket` -

0 commit comments

Comments
 (0)