Skip to content

Commit a882d62

Browse files
gurusai-voletigurusai-voleti
authored
feat: (storagetransfer) add metadata_options in transfer spec (#13371)
1 parent 87a3ada commit a882d62

File tree

2 files changed

+463
-5
lines changed

2 files changed

+463
-5
lines changed

‎mmv1/third_party/terraform/services/storagetransfer/resource_storage_transfer_job.go.tmpl

+170-3
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,38 @@ var (
4141
"transfer_spec.0.transfer_options.0.delete_objects_unique_in_sink",
4242
"transfer_spec.0.transfer_options.0.delete_objects_from_source_after_transfer",
4343
"transfer_spec.0.transfer_options.0.overwrite_when",
44+
"transfer_spec.0.transfer_options.0.metadata_options",
4445
}
4546
replicationSpecTransferOptionsKeys = []string{
4647
"replication_spec.0.transfer_options.0.overwrite_objects_already_existing_in_sink",
4748
"replication_spec.0.transfer_options.0.delete_objects_unique_in_sink",
4849
"replication_spec.0.transfer_options.0.delete_objects_from_source_after_transfer",
4950
"replication_spec.0.transfer_options.0.overwrite_when",
51+
"replication_spec.0.transfer_options.0.metadata_options",
52+
}
53+
54+
transferSpecMetadataOptionsKeys = []string{
55+
"transfer_spec.0.transfer_options.0.metadata_options.0.time_created",
56+
"transfer_spec.0.transfer_options.0.metadata_options.0.symlink",
57+
"transfer_spec.0.transfer_options.0.metadata_options.0.mode",
58+
"transfer_spec.0.transfer_options.0.metadata_options.0.gid",
59+
"transfer_spec.0.transfer_options.0.metadata_options.0.uid",
60+
"transfer_spec.0.transfer_options.0.metadata_options.0.acl",
61+
"transfer_spec.0.transfer_options.0.metadata_options.0.storage_class",
62+
"transfer_spec.0.transfer_options.0.metadata_options.0.temporary_hold",
63+
"transfer_spec.0.transfer_options.0.metadata_options.0.kms_key",
64+
}
65+
66+
replicationSpecMetadataOptionsKeys = []string{
67+
"replication_spec.0.transfer_options.0.metadata_options.0.symlink",
68+
"replication_spec.0.transfer_options.0.metadata_options.0.mode",
69+
"replication_spec.0.transfer_options.0.metadata_options.0.gid",
70+
"replication_spec.0.transfer_options.0.metadata_options.0.uid",
71+
"replication_spec.0.transfer_options.0.metadata_options.0.acl",
72+
"replication_spec.0.transfer_options.0.metadata_options.0.storage_class",
73+
"replication_spec.0.transfer_options.0.metadata_options.0.temporary_hold",
74+
"replication_spec.0.transfer_options.0.metadata_options.0.kms_key",
75+
"replication_spec.0.transfer_options.0.metadata_options.0.time_created",
5076
}
5177

5278
transferSpecDataSourceKeys = []string{
@@ -152,7 +178,7 @@ func ResourceStorageTransferJob() *schema.Resource {
152178
Elem: &schema.Resource{
153179
Schema: map[string]*schema.Schema{
154180
"object_conditions": objectConditionsSchema(replicationSpecObjectConditionsKeys),
155-
"transfer_options": transferOptionsSchema(replicationSpecTransferOptionsKeys),
181+
"transfer_options": transferOptionsSchema(replicationSpecTransferOptionsKeys, replicationSpecMetadataOptionsKeys),
156182
"gcs_data_sink": {
157183
Type: schema.TypeList,
158184
Optional: true,
@@ -182,7 +208,7 @@ func ResourceStorageTransferJob() *schema.Resource {
182208
Elem: &schema.Resource{
183209
Schema: map[string]*schema.Schema{
184210
"object_conditions": objectConditionsSchema(transferSpecObjectConditionsKeys),
185-
"transfer_options": transferOptionsSchema(transferSpecTransferOptionsKeys),
211+
"transfer_options": transferOptionsSchema(transferSpecTransferOptionsKeys, transferSpecMetadataOptionsKeys),
186212
"source_agent_pool_name": {
187213
Type: schema.TypeString,
188214
Optional: true,
@@ -459,7 +485,7 @@ func objectConditionsSchema(objectConditionsKeys []string) *schema.Schema {
459485
}
460486
}
461487

462-
func transferOptionsSchema(transferOptionsKeys []string) *schema.Schema {
488+
func transferOptionsSchema(transferOptionsKeys []string, metadataOptionsKeys []string) *schema.Schema {
463489
return &schema.Schema{
464490
Type: schema.TypeList,
465491
Optional: true,
@@ -493,12 +519,114 @@ func transferOptionsSchema(transferOptionsKeys []string) *schema.Schema {
493519
ValidateFunc: validation.StringInSlice([]string{"DIFFERENT", "NEVER", "ALWAYS"}, false),
494520
Description: `When to overwrite objects that already exist in the sink. If not set, overwrite behavior is determined by overwriteObjectsAlreadyExistingInSink.`,
495521
},
522+
"metadata_options": {
523+
Type: schema.TypeList,
524+
Optional: true,
525+
MaxItems: 1,
526+
AtLeastOneOf: transferOptionsKeys,
527+
Elem: metadataOptionsSchema(metadataOptionsKeys),
528+
Description: `Specifies the metadata options for running a transfer`,
529+
},
496530
},
497531
},
498532
Description: `Characteristics of how to treat files from datasource and sink during job. If the option delete_objects_unique_in_sink is true, object conditions based on objects' last_modification_time are ignored and do not exclude objects in a data source or a data sink.`,
499533
}
500534
}
501535

536+
func metadataOptionsSchema(metadataOptionsKeys []string) *schema.Resource {
537+
return &schema.Resource{
538+
Schema: map[string]*schema.Schema{
539+
"symlink": {
540+
Type: schema.TypeString,
541+
Optional: true,
542+
Description: "Specifies how symlinks should be handled by the transfer.",
543+
ValidateFunc: validation.StringInSlice([]string{
544+
"SYMLINK_UNSPECIFIED", "SYMLINK_SKIP", "SYMLINK_PRESERVE",
545+
}, false),
546+
AtLeastOneOf: metadataOptionsKeys,
547+
},
548+
"mode": {
549+
Type: schema.TypeString,
550+
Optional: true,
551+
Description: "Specifies how each file's mode attribute should be handled by the transfer.",
552+
ValidateFunc: validation.StringInSlice([]string{
553+
"MODE_UNSPECIFIED", "MODE_SKIP", "MODE_PRESERVE",
554+
}, false),
555+
AtLeastOneOf: metadataOptionsKeys,
556+
},
557+
"gid": {
558+
Type: schema.TypeString,
559+
Optional: true,
560+
Description: "Specifies how each file's POSIX group ID (GID) attribute should be handled by the transfer.",
561+
ValidateFunc: validation.StringInSlice([]string{
562+
"GID_UNSPECIFIED", "GID_SKIP", "GID_NUMBER",
563+
}, false),
564+
AtLeastOneOf: metadataOptionsKeys,
565+
},
566+
"uid": {
567+
Type: schema.TypeString,
568+
Optional: true,
569+
Description: "Specifies how each file's POSIX user ID (UID) attribute should be handled by the transfer.",
570+
ValidateFunc: validation.StringInSlice([]string{
571+
"UID_UNSPECIFIED", "UID_SKIP", "UID_NUMBER",
572+
}, false),
573+
AtLeastOneOf: metadataOptionsKeys,
574+
},
575+
"acl": {
576+
Type: schema.TypeString,
577+
Optional: true,
578+
Description: "Specifies how each object's ACLs should be preserved for transfers between Google Cloud Storage buckets",
579+
ValidateFunc: validation.StringInSlice([]string{
580+
"ACL_UNSPECIFIED", "ACL_DESTINATION_BUCKET_DEFAULT", "ACL_PRESERVE",
581+
}, false),
582+
AtLeastOneOf: metadataOptionsKeys,
583+
},
584+
"storage_class": {
585+
Type: schema.TypeString,
586+
Optional: true,
587+
Description: "Specifies the storage class to set on objects being transferred to Google Cloud Storage buckets",
588+
ValidateFunc: validation.StringInSlice([]string{
589+
"STORAGE_CLASS_UNSPECIFIED",
590+
"STORAGE_CLASS_DESTINATION_BUCKET_DEFAULT",
591+
"STORAGE_CLASS_PRESERVE",
592+
"STORAGE_CLASS_STANDARD",
593+
"STORAGE_CLASS_NEARLINE",
594+
"STORAGE_CLASS_COLDLINE",
595+
"STORAGE_CLASS_ARCHIVE",
596+
}, false),
597+
AtLeastOneOf: metadataOptionsKeys,
598+
},
599+
"temporary_hold": {
600+
Type: schema.TypeString,
601+
Optional: true,
602+
Description: "SSpecifies how each object's temporary hold status should be preserved for transfers between Google Cloud Storage buckets",
603+
ValidateFunc: validation.StringInSlice([]string{
604+
"TEMPORARY_HOLD_UNSPECIFIED", "TEMPORARY_HOLD_SKIP", "TEMPORARY_HOLD_PRESERVE",
605+
}, false),
606+
AtLeastOneOf: metadataOptionsKeys,
607+
},
608+
"kms_key": {
609+
Type: schema.TypeString,
610+
Optional: true,
611+
Description: "Specifies how each object's Cloud KMS customer-managed encryption key (CMEK) is preserved for transfers between Google Cloud Storage buckets",
612+
ValidateFunc: validation.StringInSlice([]string{
613+
"KMS_KEY_UNSPECIFIED", "KMS_KEY_DESTINATION_BUCKET_DEFAULT", "KMS_KEY_PRESERVE",
614+
}, false),
615+
AtLeastOneOf: metadataOptionsKeys,
616+
},
617+
"time_created": {
618+
Type: schema.TypeString,
619+
Optional: true,
620+
Description: "Specifies how each object's timeCreated metadata is preserved for transfers.",
621+
ValidateFunc: validation.StringInSlice([]string{
622+
"TIME_CREATED_UNSPECIFIED", "TIME_CREATED_SKIP", "TIME_CREATED_PRESERVE_AS_CUSTOM_TIME",
623+
}, false),
624+
AtLeastOneOf: metadataOptionsKeys,
625+
},
626+
},
627+
}
628+
}
629+
502630
func timeObjectSchema() *schema.Resource {
503631
return &schema.Resource{
504632
Schema: map[string]*schema.Schema{
@@ -1359,6 +1487,26 @@ func expandTransferOptions(options []interface{}) *storagetransfer.TransferOptio
13591487
DeleteObjectsUniqueInSink: option["delete_objects_unique_in_sink"].(bool),
13601488
OverwriteObjectsAlreadyExistingInSink: option["overwrite_objects_already_existing_in_sink"].(bool),
13611489
OverwriteWhen: option["overwrite_when"].(string),
1490+
MetadataOptions: expandMetadataOptions(option["metadata_options"].([]interface{})),
1491+
}
1492+
}
1493+
1494+
func expandMetadataOptions(options []interface{}) *storagetransfer.MetadataOptions {
1495+
if len(options) == 0 || options == nil {
1496+
return nil
1497+
}
1498+
1499+
option := options[0].(map[string]interface{})
1500+
return &storagetransfer.MetadataOptions{
1501+
Symlink: option["symlink"].(string),
1502+
Uid: option["uid"].(string),
1503+
Gid: option["gid"].(string),
1504+
Mode: option["mode"].(string),
1505+
Acl: option["acl"].(string),
1506+
StorageClass: option["storage_class"].(string),
1507+
TemporaryHold: option["temporary_hold"].(string),
1508+
KmsKey: option["kms_key"].(string),
1509+
TimeCreated: option["time_created"].(string),
13621510
}
13631511
}
13641512

@@ -1368,11 +1516,30 @@ func flattenTransferOption(option *storagetransfer.TransferOptions) []map[string
13681516
"delete_objects_unique_in_sink": option.DeleteObjectsUniqueInSink,
13691517
"overwrite_objects_already_existing_in_sink": option.OverwriteObjectsAlreadyExistingInSink,
13701518
"overwrite_when": option.OverwriteWhen,
1519+
"metadata_options": flattenMetadataOptions(option.MetadataOptions),
13711520
}
13721521

13731522
return []map[string]interface{}{data}
13741523
}
13751524

1525+
func flattenMetadataOptions(options *storagetransfer.MetadataOptions) []map[string]interface{} {
1526+
if options == nil {
1527+
return nil
1528+
}
1529+
data := map[string]interface{}{
1530+
"symlink": options.Symlink,
1531+
"uid": options.Uid,
1532+
"mode": options.Mode,
1533+
"gid": options.Gid,
1534+
"acl": options.Acl,
1535+
"storage_class": options.StorageClass,
1536+
"temporary_hold": options.TemporaryHold,
1537+
"kms_key": options.KmsKey,
1538+
"time_created": options.TimeCreated,
1539+
}
1540+
return []map[string]interface{}{data}
1541+
}
1542+
13761543
func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferSpec {
13771544
if len(transferSpecs) == 0 || transferSpecs[0] == nil {
13781545
return nil

0 commit comments

Comments
 (0)