Skip to content

Commit e518d84

Browse files
modular-magicianmodular-magician
authored
feat: (storagetransfer) add metadata_options in transfer spec (#13371) (#9567)
[upstream:a882d625144c96fa59454a434b873baf96d34081] Signed-off-by: Modular Magician <[email protected]>
1 parent 87bb3b6 commit e518d84

File tree

3 files changed

+466
-5
lines changed

3 files changed

+466
-5
lines changed

‎.changelog/13371.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
storage: added metadata_options in transfer_spec
3+
```

‎google-beta/services/storagetransfer/resource_storage_transfer_job.go

+170-3
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,38 @@ var (
4343
"transfer_spec.0.transfer_options.0.delete_objects_unique_in_sink",
4444
"transfer_spec.0.transfer_options.0.delete_objects_from_source_after_transfer",
4545
"transfer_spec.0.transfer_options.0.overwrite_when",
46+
"transfer_spec.0.transfer_options.0.metadata_options",
4647
}
4748
replicationSpecTransferOptionsKeys = []string{
4849
"replication_spec.0.transfer_options.0.overwrite_objects_already_existing_in_sink",
4950
"replication_spec.0.transfer_options.0.delete_objects_unique_in_sink",
5051
"replication_spec.0.transfer_options.0.delete_objects_from_source_after_transfer",
5152
"replication_spec.0.transfer_options.0.overwrite_when",
53+
"replication_spec.0.transfer_options.0.metadata_options",
54+
}
55+
56+
transferSpecMetadataOptionsKeys = []string{
57+
"transfer_spec.0.transfer_options.0.metadata_options.0.time_created",
58+
"transfer_spec.0.transfer_options.0.metadata_options.0.symlink",
59+
"transfer_spec.0.transfer_options.0.metadata_options.0.mode",
60+
"transfer_spec.0.transfer_options.0.metadata_options.0.gid",
61+
"transfer_spec.0.transfer_options.0.metadata_options.0.uid",
62+
"transfer_spec.0.transfer_options.0.metadata_options.0.acl",
63+
"transfer_spec.0.transfer_options.0.metadata_options.0.storage_class",
64+
"transfer_spec.0.transfer_options.0.metadata_options.0.temporary_hold",
65+
"transfer_spec.0.transfer_options.0.metadata_options.0.kms_key",
66+
}
67+
68+
replicationSpecMetadataOptionsKeys = []string{
69+
"replication_spec.0.transfer_options.0.metadata_options.0.symlink",
70+
"replication_spec.0.transfer_options.0.metadata_options.0.mode",
71+
"replication_spec.0.transfer_options.0.metadata_options.0.gid",
72+
"replication_spec.0.transfer_options.0.metadata_options.0.uid",
73+
"replication_spec.0.transfer_options.0.metadata_options.0.acl",
74+
"replication_spec.0.transfer_options.0.metadata_options.0.storage_class",
75+
"replication_spec.0.transfer_options.0.metadata_options.0.temporary_hold",
76+
"replication_spec.0.transfer_options.0.metadata_options.0.kms_key",
77+
"replication_spec.0.transfer_options.0.metadata_options.0.time_created",
5278
}
5379

5480
transferSpecDataSourceKeys = []string{
@@ -152,7 +178,7 @@ func ResourceStorageTransferJob() *schema.Resource {
152178
Elem: &schema.Resource{
153179
Schema: map[string]*schema.Schema{
154180
"object_conditions": objectConditionsSchema(replicationSpecObjectConditionsKeys),
155-
"transfer_options": transferOptionsSchema(replicationSpecTransferOptionsKeys),
181+
"transfer_options": transferOptionsSchema(replicationSpecTransferOptionsKeys, replicationSpecMetadataOptionsKeys),
156182
"gcs_data_sink": {
157183
Type: schema.TypeList,
158184
Optional: true,
@@ -182,7 +208,7 @@ func ResourceStorageTransferJob() *schema.Resource {
182208
Elem: &schema.Resource{
183209
Schema: map[string]*schema.Schema{
184210
"object_conditions": objectConditionsSchema(transferSpecObjectConditionsKeys),
185-
"transfer_options": transferOptionsSchema(transferSpecTransferOptionsKeys),
211+
"transfer_options": transferOptionsSchema(transferSpecTransferOptionsKeys, transferSpecMetadataOptionsKeys),
186212
"source_agent_pool_name": {
187213
Type: schema.TypeString,
188214
Optional: true,
@@ -459,7 +485,7 @@ func objectConditionsSchema(objectConditionsKeys []string) *schema.Schema {
459485
}
460486
}
461487

462-
func transferOptionsSchema(transferOptionsKeys []string) *schema.Schema {
488+
func transferOptionsSchema(transferOptionsKeys []string, metadataOptionsKeys []string) *schema.Schema {
463489
return &schema.Schema{
464490
Type: schema.TypeList,
465491
Optional: true,
@@ -493,12 +519,114 @@ func transferOptionsSchema(transferOptionsKeys []string) *schema.Schema {
493519
ValidateFunc: validation.StringInSlice([]string{"DIFFERENT", "NEVER", "ALWAYS"}, false),
494520
Description: `When to overwrite objects that already exist in the sink. If not set, overwrite behavior is determined by overwriteObjectsAlreadyExistingInSink.`,
495521
},
522+
"metadata_options": {
523+
Type: schema.TypeList,
524+
Optional: true,
525+
MaxItems: 1,
526+
AtLeastOneOf: transferOptionsKeys,
527+
Elem: metadataOptionsSchema(metadataOptionsKeys),
528+
Description: `Specifies the metadata options for running a transfer`,
529+
},
496530
},
497531
},
498532
Description: `Characteristics of how to treat files from datasource and sink during job. If the option delete_objects_unique_in_sink is true, object conditions based on objects' last_modification_time are ignored and do not exclude objects in a data source or a data sink.`,
499533
}
500534
}
501535

536+
func metadataOptionsSchema(metadataOptionsKeys []string) *schema.Resource {
537+
return &schema.Resource{
538+
Schema: map[string]*schema.Schema{
539+
"symlink": {
540+
Type: schema.TypeString,
541+
Optional: true,
542+
Description: "Specifies how symlinks should be handled by the transfer.",
543+
ValidateFunc: validation.StringInSlice([]string{
544+
"SYMLINK_UNSPECIFIED", "SYMLINK_SKIP", "SYMLINK_PRESERVE",
545+
}, false),
546+
AtLeastOneOf: metadataOptionsKeys,
547+
},
548+
"mode": {
549+
Type: schema.TypeString,
550+
Optional: true,
551+
Description: "Specifies how each file's mode attribute should be handled by the transfer.",
552+
ValidateFunc: validation.StringInSlice([]string{
553+
"MODE_UNSPECIFIED", "MODE_SKIP", "MODE_PRESERVE",
554+
}, false),
555+
AtLeastOneOf: metadataOptionsKeys,
556+
},
557+
"gid": {
558+
Type: schema.TypeString,
559+
Optional: true,
560+
Description: "Specifies how each file's POSIX group ID (GID) attribute should be handled by the transfer.",
561+
ValidateFunc: validation.StringInSlice([]string{
562+
"GID_UNSPECIFIED", "GID_SKIP", "GID_NUMBER",
563+
}, false),
564+
AtLeastOneOf: metadataOptionsKeys,
565+
},
566+
"uid": {
567+
Type: schema.TypeString,
568+
Optional: true,
569+
Description: "Specifies how each file's POSIX user ID (UID) attribute should be handled by the transfer.",
570+
ValidateFunc: validation.StringInSlice([]string{
571+
"UID_UNSPECIFIED", "UID_SKIP", "UID_NUMBER",
572+
}, false),
573+
AtLeastOneOf: metadataOptionsKeys,
574+
},
575+
"acl": {
576+
Type: schema.TypeString,
577+
Optional: true,
578+
Description: "Specifies how each object's ACLs should be preserved for transfers between Google Cloud Storage buckets",
579+
ValidateFunc: validation.StringInSlice([]string{
580+
"ACL_UNSPECIFIED", "ACL_DESTINATION_BUCKET_DEFAULT", "ACL_PRESERVE",
581+
}, false),
582+
AtLeastOneOf: metadataOptionsKeys,
583+
},
584+
"storage_class": {
585+
Type: schema.TypeString,
586+
Optional: true,
587+
Description: "Specifies the storage class to set on objects being transferred to Google Cloud Storage buckets",
588+
ValidateFunc: validation.StringInSlice([]string{
589+
"STORAGE_CLASS_UNSPECIFIED",
590+
"STORAGE_CLASS_DESTINATION_BUCKET_DEFAULT",
591+
"STORAGE_CLASS_PRESERVE",
592+
"STORAGE_CLASS_STANDARD",
593+
"STORAGE_CLASS_NEARLINE",
594+
"STORAGE_CLASS_COLDLINE",
595+
"STORAGE_CLASS_ARCHIVE",
596+
}, false),
597+
AtLeastOneOf: metadataOptionsKeys,
598+
},
599+
"temporary_hold": {
600+
Type: schema.TypeString,
601+
Optional: true,
602+
Description: "SSpecifies how each object's temporary hold status should be preserved for transfers between Google Cloud Storage buckets",
603+
ValidateFunc: validation.StringInSlice([]string{
604+
"TEMPORARY_HOLD_UNSPECIFIED", "TEMPORARY_HOLD_SKIP", "TEMPORARY_HOLD_PRESERVE",
605+
}, false),
606+
AtLeastOneOf: metadataOptionsKeys,
607+
},
608+
"kms_key": {
609+
Type: schema.TypeString,
610+
Optional: true,
611+
Description: "Specifies how each object's Cloud KMS customer-managed encryption key (CMEK) is preserved for transfers between Google Cloud Storage buckets",
612+
ValidateFunc: validation.StringInSlice([]string{
613+
"KMS_KEY_UNSPECIFIED", "KMS_KEY_DESTINATION_BUCKET_DEFAULT", "KMS_KEY_PRESERVE",
614+
}, false),
615+
AtLeastOneOf: metadataOptionsKeys,
616+
},
617+
"time_created": {
618+
Type: schema.TypeString,
619+
Optional: true,
620+
Description: "Specifies how each object's timeCreated metadata is preserved for transfers.",
621+
ValidateFunc: validation.StringInSlice([]string{
622+
"TIME_CREATED_UNSPECIFIED", "TIME_CREATED_SKIP", "TIME_CREATED_PRESERVE_AS_CUSTOM_TIME",
623+
}, false),
624+
AtLeastOneOf: metadataOptionsKeys,
625+
},
626+
},
627+
}
628+
}
629+
502630
func timeObjectSchema() *schema.Resource {
503631
return &schema.Resource{
504632
Schema: map[string]*schema.Schema{
@@ -1347,6 +1475,26 @@ func expandTransferOptions(options []interface{}) *storagetransfer.TransferOptio
13471475
DeleteObjectsUniqueInSink: option["delete_objects_unique_in_sink"].(bool),
13481476
OverwriteObjectsAlreadyExistingInSink: option["overwrite_objects_already_existing_in_sink"].(bool),
13491477
OverwriteWhen: option["overwrite_when"].(string),
1478+
MetadataOptions: expandMetadataOptions(option["metadata_options"].([]interface{})),
1479+
}
1480+
}
1481+
1482+
func expandMetadataOptions(options []interface{}) *storagetransfer.MetadataOptions {
1483+
if len(options) == 0 || options == nil {
1484+
return nil
1485+
}
1486+
1487+
option := options[0].(map[string]interface{})
1488+
return &storagetransfer.MetadataOptions{
1489+
Symlink: option["symlink"].(string),
1490+
Uid: option["uid"].(string),
1491+
Gid: option["gid"].(string),
1492+
Mode: option["mode"].(string),
1493+
Acl: option["acl"].(string),
1494+
StorageClass: option["storage_class"].(string),
1495+
TemporaryHold: option["temporary_hold"].(string),
1496+
KmsKey: option["kms_key"].(string),
1497+
TimeCreated: option["time_created"].(string),
13501498
}
13511499
}
13521500

@@ -1356,11 +1504,30 @@ func flattenTransferOption(option *storagetransfer.TransferOptions) []map[string
13561504
"delete_objects_unique_in_sink": option.DeleteObjectsUniqueInSink,
13571505
"overwrite_objects_already_existing_in_sink": option.OverwriteObjectsAlreadyExistingInSink,
13581506
"overwrite_when": option.OverwriteWhen,
1507+
"metadata_options": flattenMetadataOptions(option.MetadataOptions),
13591508
}
13601509

13611510
return []map[string]interface{}{data}
13621511
}
13631512

1513+
func flattenMetadataOptions(options *storagetransfer.MetadataOptions) []map[string]interface{} {
1514+
if options == nil {
1515+
return nil
1516+
}
1517+
data := map[string]interface{}{
1518+
"symlink": options.Symlink,
1519+
"uid": options.Uid,
1520+
"mode": options.Mode,
1521+
"gid": options.Gid,
1522+
"acl": options.Acl,
1523+
"storage_class": options.StorageClass,
1524+
"temporary_hold": options.TemporaryHold,
1525+
"kms_key": options.KmsKey,
1526+
"time_created": options.TimeCreated,
1527+
}
1528+
return []map[string]interface{}{data}
1529+
}
1530+
13641531
func expandTransferSpecs(transferSpecs []interface{}) *storagetransfer.TransferSpec {
13651532
if len(transferSpecs) == 0 || transferSpecs[0] == nil {
13661533
return nil

0 commit comments

Comments
 (0)