Skip to content

Commit 071a5d2

Browse files
feat: add bigtable change stream retention (#8302) (#15152)
Signed-off-by: Modular Magician <[email protected]>
1 parent 3d329f7 commit 071a5d2

File tree

6 files changed

+181
-6
lines changed

6 files changed

+181
-6
lines changed

.changelog/8302.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
bigtable: added `change_stream_retention` field to `google_bigtable_table.table` resource
3+
```

.teamcity/components/generated/services.kt

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ var services = mapOf(
5656
"essentialcontacts" to "Essentialcontacts",
5757
"filestore" to "Filestore",
5858
"firebase" to "Firebase",
59+
"firebasehosting" to "Firebasehosting",
5960
"firestore" to "Firestore",
6061
"gameservices" to "Gameservices",
6162
"gkebackup" to "Gkebackup",

go.mod

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ module github.com/hashicorp/terraform-provider-google
22
go 1.19
33

44
require (
5-
cloud.google.com/go/bigtable v1.17.0
5+
cloud.google.com/go/bigtable v1.19.0
66
github.com/GoogleCloudPlatform/declarative-resource-client-library v1.44.0
77
github.com/apparentlymart/go-cidr v1.1.0
88
github.com/davecgh/go-spew v1.1.1
@@ -33,11 +33,11 @@ require (
3333

3434
require (
3535
bitbucket.org/creachadair/stringset v0.0.8 // indirect
36-
cloud.google.com/go v0.110.0 // indirect
36+
cloud.google.com/go v0.110.2 // indirect
3737
cloud.google.com/go/compute v1.19.3 // indirect
3838
cloud.google.com/go/compute/metadata v0.2.3 // indirect
39-
cloud.google.com/go/iam v0.13.0 // indirect
40-
cloud.google.com/go/longrunning v0.4.1 // indirect
39+
cloud.google.com/go/iam v1.1.0 // indirect
40+
cloud.google.com/go/longrunning v0.5.0 // indirect
4141
github.com/agext/levenshtein v1.2.2 // indirect
4242
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
4343
github.com/cenkalti/backoff v2.2.1+incompatible // indirect

go.sum

+14-2
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,22 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
44
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
55
cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
66
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
7-
cloud.google.com/go/bigtable v1.17.0 h1:8t48YTxxFsYKy+AWuHdoePgAr4J2gEtntbdWclbEbco=
8-
cloud.google.com/go/bigtable v1.17.0/go.mod h1:wtf7lFV1Wa5ay6aKa/gv/T2Ci7J6qXpBX8Ofij2z5mo=
7+
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
8+
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
9+
cloud.google.com/go/bigtable v1.19.0 h1:wiq9LT0kukfInzvy1joMDijCw/OD1UChpSbORXYn0LI=
10+
cloud.google.com/go/bigtable v1.19.0/go.mod h1:xl5kPa8PTkJjdBxg6qdGH88464nNqmbISHSRU+D2yFE=
911
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
1012
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
1113
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
1214
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
1315
cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
1416
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
17+
cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
18+
cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
1519
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
1620
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
21+
cloud.google.com/go/longrunning v0.5.0 h1:DK8BH0+hS+DIvc9a2TPnteUievsTCH4ORMAASSb7JcQ=
22+
cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc=
1723
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
1824
github.com/GoogleCloudPlatform/declarative-resource-client-library v1.44.0 h1:hASUAck0/5j84kejIHGJjipjUzFHiN5edNMobKwj2HA=
1925
github.com/GoogleCloudPlatform/declarative-resource-client-library v1.44.0/go.mod h1:pL2Qt5HT+x6xrTd806oMiM3awW6kNIXB/iiuClz6m6k=
@@ -144,6 +150,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
144150
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
145151
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
146152
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
153+
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
154+
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
147155
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
148156
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
149157
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
@@ -198,6 +206,7 @@ github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKL
198206
github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
199207
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
200208
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
209+
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
201210
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
202211
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
203212
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -258,6 +267,7 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
258267
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
259268
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
260269
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
270+
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
261271
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
262272
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
263273
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -312,6 +322,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
312322
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
313323
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
314324
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
325+
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
315326
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
316327
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
317328
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -465,3 +476,4 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
465476
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
466477
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
467478
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
479+
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

google/resource_bigtable_table_test.go

+118
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,69 @@ func TestAccBigtableTable_deletion_protection_unprotected(t *testing.T) {
211211
})
212212
}
213213

214+
func TestAccBigtableTable_change_stream_enable(t *testing.T) {
215+
// bigtable instance does not use the shared HTTP client, this test creates an instance
216+
acctest.SkipIfVcr(t)
217+
t.Parallel()
218+
219+
instanceName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
220+
tableName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
221+
family := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10))
222+
223+
acctest.VcrTest(t, resource.TestCase{
224+
PreCheck: func() { acctest.AccTestPreCheck(t) },
225+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
226+
CheckDestroy: testAccCheckBigtableTableDestroyProducer(t),
227+
Steps: []resource.TestStep{
228+
// creating a table with a column family and change stream of 1 day
229+
{
230+
Config: testAccBigtableTable_change_stream_retention(instanceName, tableName, "24h0m0s", family),
231+
},
232+
{
233+
ResourceName: "google_bigtable_table.table",
234+
ImportState: true,
235+
ImportStateVerify: true,
236+
},
237+
// it is not possible to delete the table because of change stream is enabled
238+
{
239+
Config: testAccBigtableTable_destroyTable(instanceName),
240+
ExpectError: regexp.MustCompile(".*the change stream is enabled.*"),
241+
},
242+
// changing change stream retention value
243+
{
244+
Config: testAccBigtableTable_change_stream_retention(instanceName, tableName, "120h0m0s", family),
245+
},
246+
{
247+
ResourceName: "google_bigtable_table.table",
248+
ImportState: true,
249+
ImportStateVerify: true,
250+
},
251+
// it is not possible to delete the table because of change stream is enabled
252+
{
253+
Config: testAccBigtableTable_destroyTable(instanceName),
254+
ExpectError: regexp.MustCompile(".*the change stream is enabled.*"),
255+
},
256+
// disable changing change stream retention
257+
{
258+
Config: testAccBigtableTable_change_stream_retention(instanceName, tableName, "0", family),
259+
Check: resource.ComposeTestCheckFunc(
260+
testAccBigtableChangeStreamDisabled(t),
261+
),
262+
},
263+
// destroying the table is possible when change stream is disabled
264+
{
265+
Config: testAccBigtableTable_destroyTable(instanceName),
266+
},
267+
{
268+
ResourceName: "google_bigtable_instance.instance",
269+
ImportState: true,
270+
ImportStateVerify: true,
271+
ImportStateVerifyIgnore: []string{"deletion_protection", "instance_type"},
272+
},
273+
},
274+
})
275+
}
276+
214277
func TestAccBigtableTable_familyMany(t *testing.T) {
215278
// bigtable instance does not use the shared HTTP client, this test creates an instance
216279
acctest.SkipIfVcr(t)
@@ -328,6 +391,35 @@ func testAccBigtableColumnFamilyExists(t *testing.T, table_name_space, family st
328391
}
329392
}
330393

394+
func testAccBigtableChangeStreamDisabled(t *testing.T) resource.TestCheckFunc {
395+
var ctx = context.Background()
396+
return func(s *terraform.State) error {
397+
rs, ok := s.RootModule().Resources["google_bigtable_table.table"]
398+
if !ok {
399+
return fmt.Errorf("Table not found: %s", "google_bigtable_table.table")
400+
}
401+
402+
config := acctest.GoogleProviderConfig(t)
403+
c, err := config.BigTableClientFactory(config.UserAgent).NewAdminClient(config.Project, rs.Primary.Attributes["instance_name"])
404+
if err != nil {
405+
return fmt.Errorf("Error starting admin client. %s", err)
406+
}
407+
408+
defer c.Close()
409+
410+
table, err := c.TableInfo(ctx, rs.Primary.Attributes["name"])
411+
if err != nil {
412+
return fmt.Errorf("Error retrieving table. Could not find %s in %s.", rs.Primary.Attributes["name"], rs.Primary.Attributes["instance_name"])
413+
}
414+
415+
if table.ChangeStreamRetention != nil {
416+
return fmt.Errorf("Change Stream is expected to be disabled but it's not: %v", table)
417+
}
418+
419+
return nil
420+
}
421+
}
422+
331423
func testAccBigtableTable(instanceName, tableName string) string {
332424
return fmt.Sprintf(`
333425
resource "google_bigtable_instance" "instance" {
@@ -420,6 +512,32 @@ resource "google_bigtable_table" "table" {
420512
`, instanceName, instanceName, tableName, deletionProtection, family)
421513
}
422514

515+
func testAccBigtableTable_change_stream_retention(instanceName, tableName, changeStreamRetention, family string) string {
516+
return fmt.Sprintf(`
517+
resource "google_bigtable_instance" "instance" {
518+
name = "%s"
519+
520+
cluster {
521+
cluster_id = "%s"
522+
zone = "us-central1-b"
523+
}
524+
525+
instance_type = "DEVELOPMENT"
526+
deletion_protection = false
527+
}
528+
529+
resource "google_bigtable_table" "table" {
530+
name = "%s"
531+
instance_name = google_bigtable_instance.instance.name
532+
change_stream_retention = "%s"
533+
534+
column_family {
535+
family = "%s"
536+
}
537+
}
538+
`, instanceName, instanceName, tableName, changeStreamRetention, family)
539+
}
540+
423541
func testAccBigtableTable_familyMany(instanceName, tableName, family string) string {
424542
return fmt.Sprintf(`
425543
resource "google_bigtable_instance" "instance" {

google/services/bigtable/resource_bigtable_table.go

+41
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/hashicorp/terraform-provider-google/google/tpgresource"
1616
transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
17+
"github.com/hashicorp/terraform-provider-google/google/verify"
1718
)
1819

1920
func ResourceBigtableTable() *schema.Resource {
@@ -92,6 +93,14 @@ func ResourceBigtableTable() *schema.Resource {
9293
Elem: &schema.Schema{Type: schema.TypeString},
9394
Description: `A field to make the table protected against data loss i.e. when set to PROTECTED, deleting the table, the column families in the table, and the instance containing the table would be prohibited. If not provided, currently deletion protection will be set to UNPROTECTED as it is the API default value.`,
9495
},
96+
97+
"change_stream_retention": {
98+
Type: schema.TypeString,
99+
Optional: true,
100+
Computed: true,
101+
ValidateFunc: verify.ValidateDuration(),
102+
Description: `Duration to retain change stream data for the table. Set to 0 to disable.`,
103+
},
95104
},
96105
UseJSONNumber: true,
97106
}
@@ -134,6 +143,13 @@ func resourceBigtableTableCreate(d *schema.ResourceData, meta interface{}) error
134143
tblConf.DeletionProtection = bigtable.Unprotected
135144
}
136145

146+
if changeStreamRetention, ok := d.GetOk("change_stream_retention"); ok {
147+
tblConf.ChangeStreamRetention, err = time.ParseDuration(changeStreamRetention.(string))
148+
if err != nil {
149+
return fmt.Errorf("Error parsing change stream retention: %s", err)
150+
}
151+
}
152+
137153
// Set the split keys if given.
138154
if v, ok := d.GetOk("split_keys"); ok {
139155
tblConf.SplitKeys = tpgresource.ConvertStringArr(v.([]interface{}))
@@ -225,6 +241,14 @@ func resourceBigtableTableRead(d *schema.ResourceData, meta interface{}) error {
225241
} else {
226242
return fmt.Errorf("Error setting deletion_protection, it should be either PROTECTED or UNPROTECTED")
227243
}
244+
245+
changeStreamRetention := table.ChangeStreamRetention
246+
if changeStreamRetention != nil {
247+
if err := d.Set("change_stream_retention", changeStreamRetention.(time.Duration).String()); err != nil {
248+
return fmt.Errorf("Error setting change_stream_retention: %s", err)
249+
}
250+
}
251+
228252
return nil
229253
}
230254

@@ -292,6 +316,23 @@ func resourceBigtableTableUpdate(d *schema.ResourceData, meta interface{}) error
292316
}
293317
}
294318

319+
if d.HasChange("change_stream_retention") {
320+
changeStreamRetention := d.Get("change_stream_retention")
321+
changeStream, err := time.ParseDuration(changeStreamRetention.(string))
322+
if err != nil {
323+
return fmt.Errorf("Error parsing change stream retention: %s", err)
324+
}
325+
if changeStream == 0 {
326+
if err := c.UpdateTableDisableChangeStream(ctxWithTimeout, name); err != nil {
327+
return fmt.Errorf("Error disabling change stream retention in table %v: %s", name, err)
328+
}
329+
} else {
330+
if err := c.UpdateTableWithChangeStream(ctxWithTimeout, name, changeStream); err != nil {
331+
return fmt.Errorf("Error updating change stream retention in table %v: %s", name, err)
332+
}
333+
}
334+
}
335+
295336
return resourceBigtableTableRead(d, meta)
296337
}
297338

0 commit comments

Comments
 (0)