Skip to content

Commit 27a4918

Browse files
andreajialeeNickElliot
authored and
PerlMonker303
committed
Add AwsMsk & ConfluentCloud to 'google_pubsub_topic' (GoogleCloudPlatform#12765)
Co-authored-by: Nick Elliot <[email protected]>
1 parent c290804 commit 27a4918

File tree

4 files changed

+251
-0
lines changed

4 files changed

+251
-0
lines changed

‎mmv1/products/pubsub/Topic.yaml

+86
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ examples:
9191
primary_resource_id: 'example'
9292
vars:
9393
topic_name: 'example-topic'
94+
- name: 'pubsub_topic_ingestion_aws_msk'
95+
primary_resource_id: 'example'
96+
vars:
97+
topic_name: 'example-topic'
98+
- name: 'pubsub_topic_ingestion_confluent_cloud'
99+
primary_resource_id: 'example'
100+
vars:
101+
topic_name: 'example-topic'
94102
parameters:
95103
properties:
96104
- name: 'name'
@@ -186,6 +194,8 @@ properties:
186194
- 'aws_kinesis'
187195
- 'cloud_storage'
188196
- 'azure_event_hubs'
197+
- 'aws_msk'
198+
- 'confluent_cloud'
189199
properties:
190200
- name: 'streamArn'
191201
type: String
@@ -221,6 +231,8 @@ properties:
221231
- 'aws_kinesis'
222232
- 'cloud_storage'
223233
- 'azure_event_hubs'
234+
- 'aws_msk'
235+
- 'confluent_cloud'
224236
properties:
225237
- name: 'bucket'
226238
type: String
@@ -320,6 +332,8 @@ properties:
320332
- 'aws_kinesis'
321333
- 'cloud_storage'
322334
- 'azure_event_hubs'
335+
- 'aws_msk'
336+
- 'confluent_cloud'
323337
properties:
324338
- name: 'resourceGroup'
325339
type: String
@@ -351,3 +365,75 @@ properties:
351365
The GCP service account to be used for Federated Identity authentication
352366
with Azure (via a `AssumeRoleWithWebIdentity` call for the provided
353367
role).
368+
- name: 'awsMsk'
369+
type: NestedObject
370+
description: |
371+
Settings for ingestion from Amazon Managed Streaming for Apache Kafka.
372+
conflicts:
373+
- 'aws_kinesis'
374+
- 'cloud_storage'
375+
- 'azure_event_hubs'
376+
- 'aws_msk'
377+
- 'confluent_cloud'
378+
properties:
379+
- name: 'clusterArn'
380+
type: String
381+
description: |
382+
ARN that uniquely identifies the MSK cluster.
383+
required: true
384+
- name: 'topic'
385+
type: String
386+
description: |
387+
The name of the MSK topic that Pub/Sub will import from.
388+
required: true
389+
- name: 'awsRoleArn'
390+
type: String
391+
description: |
392+
AWS role ARN to be used for Federated Identity authentication with
393+
MSK. Check the Pub/Sub docs for how to set up this role and the
394+
required permissions that need to be attached to it.
395+
required: true
396+
- name: 'gcpServiceAccount'
397+
type: String
398+
description: |
399+
The GCP service account to be used for Federated Identity authentication
400+
with MSK (via a `AssumeRoleWithWebIdentity` call for the provided
401+
role). The `awsRoleArn` must be set up with `accounts.google.com:sub`
402+
equals to this service account number.
403+
required: true
404+
- name: 'confluentCloud'
405+
type: NestedObject
406+
description: |
407+
Settings for ingestion from Confluent Cloud.
408+
conflicts:
409+
- 'aws_kinesis'
410+
- 'cloud_storage'
411+
- 'azure_event_hubs'
412+
- 'aws_msk'
413+
- 'confluent_cloud'
414+
properties:
415+
- name: 'bootstrapServer'
416+
type: String
417+
description: |
418+
The Confluent Cloud bootstrap server. The format is url:port.
419+
required: true
420+
- name: 'clusterId'
421+
type: String
422+
description: |
423+
The Confluent Cloud cluster ID.
424+
- name: 'topic'
425+
type: String
426+
description: |
427+
Name of the Confluent Cloud topic that Pub/Sub will import from.
428+
required: true
429+
- name: 'identityPoolId'
430+
type: String
431+
description: |
432+
Identity pool ID to be used for Federated Identity authentication with Confluent Cloud.
433+
required: true
434+
- name: 'gcpServiceAccount'
435+
type: String
436+
description: |
437+
The GCP service account to be used for Federated Identity authentication
438+
with Confluent Cloud.
439+
required: true

‎mmv1/templates/terraform/examples/pubsub_topic_ingestion_aws_msk.tf.tmpl

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
resource "google_pubsub_topic" "{{$.PrimaryResourceId}}" {
2+
name = "{{index $.Vars "topic_name"}}"
3+
4+
# Outside of automated terraform-provider-google CI tests, these values must be of actual AWS resources for the test to pass.
5+
ingestion_data_source_settings {
6+
aws_msk {
7+
cluster_arn = "arn:aws:kinesis:us-west-2:111111111111:stream/fake-stream-name"
8+
topic = "test-topic"
9+
aws_role_arn = "arn:aws:iam::111111111111:role/fake-role-name"
10+
gcp_service_account = "[email protected]"
11+
}
12+
}
13+
}

‎mmv1/templates/terraform/examples/pubsub_topic_ingestion_confluent_cloud.tf.tmpl

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
resource "google_pubsub_topic" "{{$.PrimaryResourceId}}" {
2+
name = "{{index $.Vars "topic_name"}}"
3+
4+
# Outside of automated terraform-provider-google CI tests, these values must be of actual Confluent Cloud resources for the test to pass.
5+
ingestion_data_source_settings {
6+
confluent_cloud {
7+
bootstrap_server = "test.us-west2.gcp.confluent.cloud:1111"
8+
cluster_id = "1234"
9+
topic = "test-topic"
10+
identity_pool_id = "test-identity-pool-id"
11+
gcp_service_account = "[email protected]"
12+
}
13+
}
14+
}

‎mmv1/third_party/terraform/services/pubsub/resource_pubsub_topic_test.go

+138
Original file line numberDiff line numberDiff line change
@@ -445,3 +445,141 @@ resource "google_pubsub_topic" "foo" {
445445
}
446446
`, topic)
447447
}
448+
449+
func TestAccPubsubTopic_awsMskIngestionUpdate(t *testing.T) {
450+
t.Parallel()
451+
452+
topic := fmt.Sprintf("tf-test-topic-%s", acctest.RandString(t, 10))
453+
454+
acctest.VcrTest(t, resource.TestCase{
455+
PreCheck: func() { acctest.AccTestPreCheck(t) },
456+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
457+
CheckDestroy: testAccCheckPubsubTopicDestroyProducer(t),
458+
Steps: []resource.TestStep{
459+
{
460+
Config: testAccPubsubTopic_updateWithAwsMskIngestionSettings(topic),
461+
},
462+
{
463+
ResourceName: "google_pubsub_topic.foo",
464+
ImportStateId: topic,
465+
ImportState: true,
466+
ImportStateVerify: true,
467+
},
468+
{
469+
Config: testAccPubsubTopic_updateWithUpdatedAwsMskIngestionSettings(topic),
470+
},
471+
{
472+
ResourceName: "google_pubsub_topic.foo",
473+
ImportStateId: topic,
474+
ImportState: true,
475+
ImportStateVerify: true,
476+
},
477+
},
478+
})
479+
}
480+
481+
func testAccPubsubTopic_updateWithAwsMskIngestionSettings(topic string) string {
482+
return fmt.Sprintf(`
483+
resource "google_pubsub_topic" "foo" {
484+
name = "%s"
485+
486+
# Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass.
487+
ingestion_data_source_settings {
488+
aws_msk {
489+
cluster_arn = "arn:aws:kinesis:us-west-2:111111111111:stream/fake-stream-name"
490+
topic = "test-topic"
491+
aws_role_arn = "arn:aws:iam::111111111111:role/fake-role-name"
492+
gcp_service_account = "[email protected]"
493+
}
494+
}
495+
}
496+
`, topic)
497+
}
498+
499+
func testAccPubsubTopic_updateWithUpdatedAwsMskIngestionSettings(topic string) string {
500+
return fmt.Sprintf(`
501+
resource "google_pubsub_topic" "foo" {
502+
name = "%s"
503+
504+
# Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass.
505+
ingestion_data_source_settings {
506+
aws_msk {
507+
cluster_arn = "arn:aws:kinesis:us-west-2:111111111111:stream/fake-stream-name"
508+
topic = "test-topic"
509+
aws_role_arn = "arn:aws:iam::111111111111:role/fake-role-name"
510+
gcp_service_account = "updated-fake-service-account@fake-gcp-project.iam.gserviceaccount.com"
511+
}
512+
}
513+
}
514+
`, topic)
515+
}
516+
517+
func TestAccPubsubTopic_confluentCloudIngestionUpdate(t *testing.T) {
518+
t.Parallel()
519+
520+
topic := fmt.Sprintf("tf-test-topic-%s", acctest.RandString(t, 10))
521+
522+
acctest.VcrTest(t, resource.TestCase{
523+
PreCheck: func() { acctest.AccTestPreCheck(t) },
524+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
525+
CheckDestroy: testAccCheckPubsubTopicDestroyProducer(t),
526+
Steps: []resource.TestStep{
527+
{
528+
Config: testAccPubsubTopic_updateWithConfluentCloudIngestionSettings(topic),
529+
},
530+
{
531+
ResourceName: "google_pubsub_topic.foo",
532+
ImportStateId: topic,
533+
ImportState: true,
534+
ImportStateVerify: true,
535+
},
536+
{
537+
Config: testAccPubsubTopic_updateWithUpdatedConfluentCloudIngestionSettings(topic),
538+
},
539+
{
540+
ResourceName: "google_pubsub_topic.foo",
541+
ImportStateId: topic,
542+
ImportState: true,
543+
ImportStateVerify: true,
544+
},
545+
},
546+
})
547+
}
548+
549+
func testAccPubsubTopic_updateWithConfluentCloudIngestionSettings(topic string) string {
550+
return fmt.Sprintf(`
551+
resource "google_pubsub_topic" "foo" {
552+
name = "%s"
553+
554+
# Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass.
555+
ingestion_data_source_settings {
556+
confluent_cloud {
557+
bootstrap_server = "test.us-west2.gcp.confluent.cloud:1111"
558+
cluster_id = "1234"
559+
topic = "test-topic"
560+
identity_pool_id = "test-identity-pool-id"
561+
gcp_service_account = "[email protected]"
562+
}
563+
}
564+
}
565+
`, topic)
566+
}
567+
568+
func testAccPubsubTopic_updateWithUpdatedConfluentCloudIngestionSettings(topic string) string {
569+
return fmt.Sprintf(`
570+
resource "google_pubsub_topic" "foo" {
571+
name = "%s"
572+
573+
# Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass.
574+
ingestion_data_source_settings {
575+
confluent_cloud {
576+
bootstrap_server = "test.us-west2.gcp.confluent.cloud:1111"
577+
cluster_id = "1234"
578+
topic = "test-topic"
579+
identity_pool_id = "test-identity-pool-id"
580+
gcp_service_account = "updated-fake-service-account@fake-gcp-project.iam.gserviceaccount.com"
581+
}
582+
}
583+
}
584+
`, topic)
585+
}

0 commit comments

Comments
 (0)