Skip to content

Commit 4383e10

Browse files
modular-magicianmodular-magician
authored
Add data source for retrieving multiple objects from a GCS bucket (#10435) (#17920)
[upstream:61cec48a053ada7b72805d16098fba197cdfaafb] Signed-off-by: Modular Magician <[email protected]>
1 parent 6e13420 commit 4383e10

4 files changed

+316
-0
lines changed

‎google/provider/provider_mmv1_resources.go

+1
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ var handwrittenDatasources = map[string]*schema.Resource{
275275
"google_service_networking_peered_dns_domain": servicenetworking.DataSourceGoogleServiceNetworkingPeeredDNSDomain(),
276276
"google_storage_bucket": storage.DataSourceGoogleStorageBucket(),
277277
"google_storage_bucket_object": storage.DataSourceGoogleStorageBucketObject(),
278+
"google_storage_bucket_objects": storage.DataSourceGoogleStorageBucketObjects(),
278279
"google_storage_bucket_object_content": storage.DataSourceGoogleStorageBucketObjectContent(),
279280
"google_storage_object_signed_url": storage.DataSourceGoogleSignedUrl(),
280281
"google_storage_project_service_account": storage.DataSourceGoogleStorageProjectServiceAccount(),

‎google/services/storage/data_source_google_storage_bucket_objects.go

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// Copyright (c) HashiCorp, Inc.
2+
// SPDX-License-Identifier: MPL-2.0
3+
// Copyright (c) HashiCorp, Inc.
4+
// SPDX-License-Identifier: MPL-2.0
5+
package storage
6+
7+
import (
8+
"fmt"
9+
10+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
11+
"github.com/hashicorp/terraform-provider-google/google/tpgresource"
12+
transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
13+
)
14+
15+
func DataSourceGoogleStorageBucketObjects() *schema.Resource {
16+
return &schema.Resource{
17+
Read: datasourceGoogleStorageBucketObjectsRead,
18+
Schema: map[string]*schema.Schema{
19+
"bucket": {
20+
Type: schema.TypeString,
21+
Required: true,
22+
},
23+
"match_glob": {
24+
Type: schema.TypeString,
25+
Optional: true,
26+
},
27+
"prefix": {
28+
Type: schema.TypeString,
29+
Optional: true,
30+
},
31+
"bucket_objects": {
32+
Type: schema.TypeList,
33+
Computed: true,
34+
Elem: &schema.Resource{
35+
Schema: map[string]*schema.Schema{
36+
"content_type": {
37+
Type: schema.TypeString,
38+
Computed: true,
39+
},
40+
"media_link": {
41+
Type: schema.TypeString,
42+
Computed: true,
43+
},
44+
"name": {
45+
Type: schema.TypeString,
46+
Computed: true,
47+
},
48+
"self_link": {
49+
Type: schema.TypeString,
50+
Computed: true,
51+
},
52+
"storage_class": {
53+
Type: schema.TypeString,
54+
Computed: true,
55+
},
56+
},
57+
},
58+
},
59+
},
60+
}
61+
}
62+
63+
func datasourceGoogleStorageBucketObjectsRead(d *schema.ResourceData, meta interface{}) error {
64+
config := meta.(*transport_tpg.Config)
65+
userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
66+
if err != nil {
67+
return err
68+
}
69+
70+
params := make(map[string]string)
71+
bucketObjects := make([]map[string]interface{}, 0)
72+
73+
for {
74+
bucket := d.Get("bucket").(string)
75+
url := fmt.Sprintf("https://storage.googleapis.com/storage/v1/b/%s/o", bucket)
76+
77+
if v, ok := d.GetOk("match_glob"); ok {
78+
params["matchGlob"] = v.(string)
79+
}
80+
81+
if v, ok := d.GetOk("prefix"); ok {
82+
params["prefix"] = v.(string)
83+
}
84+
85+
url, err := transport_tpg.AddQueryParams(url, params)
86+
if err != nil {
87+
return err
88+
}
89+
90+
res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
91+
Config: config,
92+
Method: "GET",
93+
RawURL: url,
94+
UserAgent: userAgent,
95+
})
96+
if err != nil {
97+
return fmt.Errorf("Error retrieving bucket objects: %s", err)
98+
}
99+
100+
pageBucketObjects := flattenDatasourceGoogleBucketObjectsList(res["items"])
101+
bucketObjects = append(bucketObjects, pageBucketObjects...)
102+
103+
pToken, ok := res["nextPageToken"]
104+
if ok && pToken != nil && pToken.(string) != "" {
105+
params["pageToken"] = pToken.(string)
106+
} else {
107+
break
108+
}
109+
}
110+
111+
if err := d.Set("bucket_objects", bucketObjects); err != nil {
112+
return fmt.Errorf("Error retrieving bucket_objects: %s", err)
113+
}
114+
115+
d.SetId(d.Get("bucket").(string))
116+
117+
return nil
118+
}
119+
120+
func flattenDatasourceGoogleBucketObjectsList(v interface{}) []map[string]interface{} {
121+
if v == nil {
122+
return make([]map[string]interface{}, 0)
123+
}
124+
125+
ls := v.([]interface{})
126+
bucketObjects := make([]map[string]interface{}, 0, len(ls))
127+
for _, raw := range ls {
128+
o := raw.(map[string]interface{})
129+
130+
var mContentType, mMediaLink, mName, mSelfLink, mStorageClass interface{}
131+
if oContentType, ok := o["contentType"]; ok {
132+
mContentType = oContentType
133+
}
134+
if oMediaLink, ok := o["mediaLink"]; ok {
135+
mMediaLink = oMediaLink
136+
}
137+
if oName, ok := o["name"]; ok {
138+
mName = oName
139+
}
140+
if oSelfLink, ok := o["selfLink"]; ok {
141+
mSelfLink = oSelfLink
142+
}
143+
if oStorageClass, ok := o["storageClass"]; ok {
144+
mStorageClass = oStorageClass
145+
}
146+
bucketObjects = append(bucketObjects, map[string]interface{}{
147+
"content_type": mContentType,
148+
"media_link": mMediaLink,
149+
"name": mName,
150+
"self_link": mSelfLink,
151+
"storage_class": mStorageClass,
152+
})
153+
}
154+
155+
return bucketObjects
156+
}

‎google/services/storage/data_source_google_storage_bucket_objects_test.go

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright (c) HashiCorp, Inc.
2+
// SPDX-License-Identifier: MPL-2.0
3+
// Copyright (c) HashiCorp, Inc.
4+
// SPDX-License-Identifier: MPL-2.0
5+
package storage_test
6+
7+
import (
8+
"fmt"
9+
"testing"
10+
11+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
12+
"github.com/hashicorp/terraform-provider-google/google/acctest"
13+
"github.com/hashicorp/terraform-provider-google/google/envvar"
14+
)
15+
16+
func TestAccDataSourceGoogleStorageBucketObjects_basic(t *testing.T) {
17+
t.Parallel()
18+
19+
project := envvar.GetTestProjectFromEnv()
20+
bucket := "tf-bucket-object-test-" + acctest.RandString(t, 10)
21+
22+
context := map[string]interface{}{
23+
"bucket": bucket,
24+
"project": project,
25+
"object_0_name": "bee",
26+
"object_1_name": "fly",
27+
}
28+
29+
acctest.VcrTest(t, resource.TestCase{
30+
PreCheck: func() { acctest.AccTestPreCheck(t) },
31+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
32+
Steps: []resource.TestStep{
33+
{
34+
Config: testAccCheckGoogleStorageBucketObjectsConfig(context),
35+
Check: resource.ComposeTestCheckFunc(
36+
// Test schema
37+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.content_type"),
38+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.media_link"),
39+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.name"),
40+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.self_link"),
41+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.storage_class"),
42+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.content_type"),
43+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.media_link"),
44+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.name"),
45+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.self_link"),
46+
resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.storage_class"),
47+
// Test content
48+
resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket", context["bucket"].(string)),
49+
resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.name", context["object_0_name"].(string)),
50+
resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.name", context["object_1_name"].(string)),
51+
// Test match_glob
52+
resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_bee_glob", "bucket_objects.0.name", context["object_0_name"].(string)),
53+
// Test prefix
54+
resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_fly_prefix", "bucket_objects.0.name", context["object_1_name"].(string)),
55+
),
56+
},
57+
},
58+
})
59+
}
60+
61+
func testAccCheckGoogleStorageBucketObjectsConfig(context map[string]interface{}) string {
62+
return fmt.Sprintf(`
63+
resource "google_storage_bucket" "my_insect_cage" {
64+
force_destroy = true
65+
location = "EU"
66+
name = "%s"
67+
project = "%s"
68+
uniform_bucket_level_access = true
69+
}
70+
71+
resource "google_storage_bucket_object" "bee" {
72+
bucket = google_storage_bucket.my_insect_cage.name
73+
content = "bzzzzzt"
74+
name = "%s"
75+
}
76+
77+
resource "google_storage_bucket_object" "fly" {
78+
bucket = google_storage_bucket.my_insect_cage.name
79+
content = "zzzzzt"
80+
name = "%s"
81+
}
82+
83+
data "google_storage_bucket_objects" "my_insects" {
84+
bucket = google_storage_bucket.my_insect_cage.name
85+
86+
depends_on = [
87+
google_storage_bucket_object.bee,
88+
google_storage_bucket_object.fly,
89+
]
90+
}
91+
92+
data "google_storage_bucket_objects" "my_bee_glob" {
93+
bucket = google_storage_bucket.my_insect_cage.name
94+
match_glob = "b*"
95+
96+
depends_on = [
97+
google_storage_bucket_object.bee,
98+
]
99+
}
100+
101+
data "google_storage_bucket_objects" "my_fly_prefix" {
102+
bucket = google_storage_bucket.my_insect_cage.name
103+
prefix = "f"
104+
105+
depends_on = [
106+
google_storage_bucket_object.fly,
107+
]
108+
}`,
109+
context["bucket"].(string),
110+
context["project"].(string),
111+
context["object_0_name"].(string),
112+
context["object_1_name"].(string),
113+
)
114+
}

‎website/docs/d/storage_bucket_objects.html.markdown

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
---
2+
subcategory: "Cloud Storage"
3+
description: |-
4+
Retrieve information about a set of GCS bucket objects in a GCS bucket.
5+
---
6+
7+
8+
# google\_storage\_bucket\_objects
9+
10+
Gets existing objects inside an existing bucket in Google Cloud Storage service (GCS).
11+
See [the official documentation](https://cloud.google.com/storage/docs/key-terms#objects)
12+
and [API](https://cloud.google.com/storage/docs/json_api/v1/objects/list).
13+
14+
## Example Usage
15+
16+
Example files stored within a bucket.
17+
18+
```hcl
19+
data "google_storage_bucket_objects" "files" {
20+
bucket = "file-store"
21+
}
22+
```
23+
24+
## Argument Reference
25+
26+
The following arguments are supported:
27+
28+
* `bucket` - (Required) The name of the containing bucket.
29+
* `match_glob` - (Optional) A glob pattern used to filter results (for example, `foo*bar`).
30+
* `prefix` - (Optional) Filter results to include only objects whose names begin with this prefix.
31+
32+
33+
## Attributes Reference
34+
35+
The following attributes are exported:
36+
37+
* `bucket_objects` - A list of retrieved objects contained in the provided GCS bucket. Structure is [defined below](#nested_bucket_objects).
38+
39+
<a name="nested_bucket_objects"></a>The `bucket_objects` block supports:
40+
41+
* `content_type` - [Content-Type](https://tools.ietf.org/html/rfc7231#section-3.1.1.5) of the object data.
42+
* `media_link` - A url reference to download this object.
43+
* `name` - The name of the object.
44+
* `self_link` - A url reference to this object.
45+
* `storage_class` - The [StorageClass](https://cloud.google.com/storage/docs/storage-classes) of the bucket object.

0 commit comments

Comments
 (0)