Subscription.yaml

# Copyright 2024 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
name: 'Subscription'
description: |
  A named resource representing the stream of messages from a single,
  specific topic, to be delivered to the subscribing application.
references:
  guides:
    'Managing Subscriptions': 'https://cloud.google.com/pubsub/docs/admin#managing_subscriptions'
  api: 'https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions'
docs:
  note: |
    You can retrieve the email of the Google Managed Pub/Sub Service Account used for forwarding
    by using the `google_project_service_identity` resource.
base_url: 'projects/{{project}}/subscriptions'
create_verb: 'PUT'
update_url: 'projects/{{project}}/subscriptions/{{name}}'
update_verb: 'PATCH'
update_mask: true
timeouts:
  insert_minutes: 20
  update_minutes: 20
  delete_minutes: 20
async:
  type: 'PollAsync'
  check_response_func_existence: 'transport_tpg.PollCheckForExistence'
  check_response_func_absence: 'transport_tpg.PollCheckForAbsence'
  suppress_error: true
  target_occurrences: 1
  actions: ['create']
custom_code:
  constants: 'templates/terraform/constants/subscription.go.tmpl'
  encoder: 'templates/terraform/encoders/no_send_name.go.tmpl'
  update_encoder: 'templates/terraform/update_encoder/pubsub_subscription.tmpl'
examples:
  - name: 'pubsub_subscription_push'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
  - name: 'pubsub_subscription_pull'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
  - name: 'pubsub_subscription_pull_filter'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
  - name: 'pubsub_subscription_dead_letter'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
  - name: 'pubsub_subscription_push_bq'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      dataset_id: 'example_dataset'
      table_id: 'example_table'
  - name: 'pubsub_subscription_push_bq_table_schema'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      dataset_id: 'example_dataset'
      table_id: 'example_table'
  - name: 'pubsub_subscription_push_bq_service_account'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      dataset_id: 'example_dataset'
      table_id: 'example_table'
      service_account_id: 'example-bqw'
  - name: 'pubsub_subscription_push_cloudstorage'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      bucket_name: 'example-bucket'
  - name: 'pubsub_subscription_push_cloudstorage_avro'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      bucket_name: 'example-bucket'
  - name: 'pubsub_subscription_push_cloudstorage_service_account'
    primary_resource_id: 'example'
    vars:
      topic_name: 'example-topic'
      subscription_name: 'example-subscription'
      bucket_name: 'example-bucket'
      service_account_id: 'example-stw'
parameters:
properties:
  - name: 'name'
    type: String
    description: 'Name of the subscription.'
    required: true
    immutable: true
    custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.tmpl'
    custom_expand: 'templates/terraform/custom_expand/shortname_to_url.go.tmpl'
  - name: 'topic'
    type: ResourceRef
    description: |
      A reference to a Topic resource, of the form projects/{project}/topics/{{name}}
      (as in the id property of a google_pubsub_topic), or just a topic name if
      the topic is in the same project as the subscription.
    required: true
    immutable: true
    diff_suppress_func: 'tpgresource.CompareSelfLinkOrResourceName'
    custom_expand: 'templates/terraform/custom_expand/computed_subscription_topic.tmpl'
    resource: 'Topic'
    imports: 'name'
  - name: 'labels'
    type: KeyValueLabels
    description: |
      A set of key/value label pairs to assign to this Subscription.
  - name: 'bigqueryConfig'
    type: NestedObject
    description: |
      If delivery to BigQuery is used with this subscription, this field is used to configure it.
      Either pushConfig, bigQueryConfig or cloudStorageConfig can be set, but not combined.
      If all three are empty, then the subscriber will pull and ack messages using API methods.
    conflicts:
      - push_config
      - cloud_storage_config
    properties:
      - name: 'table'
        type: String
        description: |
          The name of the table to which to write data, of the form {projectId}.{datasetId}.{tableId}
        required: true
      - name: 'useTopicSchema'
        type: Boolean
        description: |
          When true, use the topic's schema as the columns to write to in BigQuery, if it exists.
          Only one of use_topic_schema and use_table_schema can be set.
        conflicts:
          - use_table_schema
      - name: 'useTableSchema'
        type: Boolean
        description: |
          When true, use the BigQuery table's schema as the columns to write to in BigQuery. Messages
          must be published in JSON format. Only one of use_topic_schema and use_table_schema can be set.
        conflicts:
          - use_topic_schema
      - name: 'writeMetadata'
        type: Boolean
        description: |
          When true, write the subscription name, messageId, publishTime, attributes, and orderingKey to additional columns in the table.
          The subscription name, messageId, and publishTime fields are put in their own columns while all other message properties (other than data) are written to a JSON object in the attributes column.
      - name: 'dropUnknownFields'
        type: Boolean
        description: |
          When true and use_topic_schema or use_table_schema is true, any fields that are a part of the topic schema or message schema that
          are not part of the BigQuery table schema are dropped when writing to BigQuery. Otherwise, the schemas must be kept in sync
          and any messages with extra fields are not written and remain in the subscription's backlog.
      - name: 'serviceAccountEmail'
        type: String
        description: |
          The service account to use to write to BigQuery. If not specified, the Pub/Sub
          [service agent](https://cloud.google.com/iam/docs/service-agents),
          service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com, is used.
  - name: 'cloudStorageConfig'
    type: NestedObject
    description: |
      If delivery to Cloud Storage is used with this subscription, this field is used to configure it.
      Either pushConfig, bigQueryConfig or cloudStorageConfig can be set, but not combined.
      If all three are empty, then the subscriber will pull and ack messages using API methods.
    conflicts:
      - push_config
      - bigquery_config
    properties:
      - name: 'bucket'
        type: String
        description: |
          User-provided name for the Cloud Storage bucket. The bucket must be created by the user. The bucket name must be without any prefix like "gs://".
        required: true
      - name: 'filenamePrefix'
        type: String
        description: |
          User-provided prefix for Cloud Storage filename.
      - name: 'filenameSuffix'
        type: String
        description: |
          User-provided suffix for Cloud Storage filename. Must not end in "/".
      - name: 'filenameDatetimeFormat'
        type: String
        description: |
          User-provided format string specifying how to represent datetimes in Cloud Storage filenames.
      - name: 'maxDuration'
        type: String
        description: |
          The maximum duration that can elapse before a new Cloud Storage file is created. Min 1 minute, max 10 minutes, default 5 minutes.
          May not exceed the subscription's acknowledgement deadline.
          A duration in seconds with up to nine fractional digits, ending with 's'. Example: "3.5s".
        default_value: "300s"
      - name: 'maxBytes'
        type: Integer
        description: |
          The maximum bytes that can be written to a Cloud Storage file before a new file is created. Min 1 KB, max 10 GiB.
          The maxBytes limit may be exceeded in cases where messages are larger than the limit.
      - name: 'maxMessages'
        type: Integer
        description: |
          The maximum messages that can be written to a Cloud Storage file before a new file is created. Min 1000 messages.
      - name: 'state'
        type: Enum
        description: |
          An output-only field that indicates whether or not the subscription can receive messages.
        output: true
        enum_values:
          - 'ACTIVE'
          - 'PERMISSION_DENIED'
          - 'NOT_FOUND'
      - name: 'avroConfig'
        type: NestedObject
        send_empty_value: true
        description: |
          If set, message data will be written to Cloud Storage in Avro format.
        properties:
          - name: 'writeMetadata'
            type: Boolean
            send_empty_value: true
            description: |
              When true, write the subscription name, messageId, publishTime, attributes, and orderingKey as additional fields in the output.
          - name: 'useTopicSchema'
            type: Boolean
            send_empty_value: true
            description: |
              When true, the output Cloud Storage file will be serialized using the topic schema, if it exists.
      - name: 'serviceAccountEmail'
        type: String
        description: |
          The service account to use to write to Cloud Storage. If not specified, the Pub/Sub
          [service agent](https://cloud.google.com/iam/docs/service-agents),
          service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com, is used.
  - name: 'pushConfig'
    type: NestedObject
    description: |
      If push delivery is used with this subscription, this field is used to
      configure it. An empty pushConfig signifies that the subscriber will
      pull and ack messages using API methods.
    conflicts:
      - bigquery_config
      - cloud_storage_config
    properties:
      - name: 'oidcToken'
        type: NestedObject
        description: |
          If specified, Pub/Sub will generate and attach an OIDC JWT token as
          an Authorization header in the HTTP request for every pushed message.
        properties:
          - name: 'serviceAccountEmail'
            type: String
            description: |
              Service account email to be used for generating the OIDC token.
              The caller (for subscriptions.create, subscriptions.patch, and
              subscriptions.modifyPushConfig RPCs) must have the
              iam.serviceAccounts.actAs permission for the service account.
            required: true
          - name: 'audience'
            type: String
            description: |
              Audience to be used when generating OIDC token. The audience claim
              identifies the recipients that the JWT is intended for. The audience
              value is a single case-sensitive string. Having multiple values (array)
              for the audience field is not supported. More info about the OIDC JWT
              token audience here: https://tools.ietf.org/html/rfc7519#section-4.1.3
              Note: if not specified, the Push endpoint URL will be used.
      - name: 'pushEndpoint'
        type: String
        description: |
          A URL locating the endpoint to which messages should be pushed.
          For example, a Webhook endpoint might use
          "https://example.com/push".
        required: true
      - name: 'attributes'
        type: KeyValuePairs
        description: |
          Endpoint configuration attributes.

          Every endpoint has a set of API supported attributes that can
          be used to control different aspects of the message delivery.

          The currently supported attribute is x-goog-version, which you
          can use to change the format of the pushed message. This
          attribute indicates the version of the data expected by
          the endpoint. This controls the shape of the pushed message
          (i.e., its fields and metadata). The endpoint version is
          based on the version of the Pub/Sub API.

          If not present during the subscriptions.create call,
          it will default to the version of the API used to make
          such call. If not present during a subscriptions.modifyPushConfig
          call, its value will not be changed. subscriptions.get
          calls will always return a valid version, even if the
          subscription was created without this attribute.

          The possible values for this attribute are:

          - v1beta1: uses the push format defined in the v1beta1 Pub/Sub API.
          - v1 or v1beta2: uses the push format defined in the v1 Pub/Sub API.
        diff_suppress_func: 'IgnoreMissingKeyInMap("x-goog-version")'
      - name: 'noWrapper'
        type: NestedObject
        description: |
          When set, the payload to the push endpoint is not wrapped.Sets the
          `data` field as the HTTP body for delivery.
        custom_flatten: 'templates/terraform/custom_flatten/pubsub_no_wrapper_write_metadata_flatten.go.tmpl'
        properties:
          - name: 'writeMetadata'
            type: Boolean
            description: |
              When true, writes the Pub/Sub message metadata to
              `x-goog-pubsub-<KEY>:<VAL>` headers of the HTTP request. Writes the
              Pub/Sub message attributes to `<KEY>:<VAL>` headers of the HTTP request.
            required: true
            send_empty_value: true
  - name: 'ackDeadlineSeconds'
    type: Integer
    description: |
      This value is the maximum time after a subscriber receives a message
      before the subscriber should acknowledge the message. After message
      delivery but before the ack deadline expires and before the message is
      acknowledged, it is an outstanding message and will not be delivered
      again during that time (on a best-effort basis).

      For pull subscriptions, this value is used as the initial value for
      the ack deadline. To override this value for a given message, call
      subscriptions.modifyAckDeadline with the corresponding ackId if using
      pull. The minimum custom deadline you can specify is 10 seconds. The
      maximum custom deadline you can specify is 600 seconds (10 minutes).
      If this parameter is 0, a default value of 10 seconds is used.

      For push delivery, this value is also used to set the request timeout
      for the call to the push endpoint.

      If the subscriber never acknowledges the message, the Pub/Sub system
      will eventually redeliver the message.
    default_from_api: true
  - name: 'messageRetentionDuration'
    type: String
    description: |
      How long to retain unacknowledged messages in the subscription's
      backlog, from the moment a message is published. If
      retain_acked_messages is true, then this also configures the retention
      of acknowledged messages, and thus configures how far back in time a
      subscriptions.seek can be done. Defaults to 7 days. Cannot be more
      than 31 days (`"2678400s"`) or less than 10 minutes (`"600s"`).

      A duration in seconds with up to nine fractional digits, terminated
      by 's'. Example: `"600.5s"`.
    default_value: "604800s"
  - name: 'retainAckedMessages'
    type: Boolean
    description: |
      Indicates whether to retain acknowledged messages. If `true`, then
      messages are not expunged from the subscription's backlog, even if
      they are acknowledged, until they fall out of the
      messageRetentionDuration window.
  - name: 'expirationPolicy'
    type: NestedObject
    description: |
      A policy that specifies the conditions for this subscription's expiration.
      A subscription is considered active as long as any connected subscriber
      is successfully consuming messages from the subscription or is issuing
      operations on the subscription. If expirationPolicy is not set, a default
      policy with ttl of 31 days will be used.  If it is set but ttl is "", the
      resource never expires.  The minimum allowed value for expirationPolicy.ttl
      is 1 day.
    default_from_api: true
    send_empty_value: true
    allow_empty_object: true
    properties:
      - name: 'ttl'
        type: String
        description: |
          Specifies the "time-to-live" duration for an associated resource. The
          resource expires if it is not active for a period of ttl.
          If ttl is set to "", the associated resource never expires.
          A duration in seconds with up to nine fractional digits, terminated by 's'.
          Example - "3.5s".
        required: true
        diff_suppress_func: 'comparePubsubSubscriptionExpirationPolicy'
  - name: 'filter'
    type: String
    description: |
      The subscription only delivers the messages that match the filter.
      Pub/Sub automatically acknowledges the messages that don't match the filter. You can filter messages
      by their attributes. The maximum length of a filter is 256 bytes. After creating the subscription,
      you can't modify the filter.
    required: false
    immutable: true
    validation:
      regex: '^[\s\S]{0,256}$'
  - name: 'deadLetterPolicy'
    type: NestedObject
    description: |
      A policy that specifies the conditions for dead lettering messages in
      this subscription. If dead_letter_policy is not set, dead lettering
      is disabled.

      The Cloud Pub/Sub service account associated with this subscription's
      parent project (i.e.,
      service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have
      permission to Acknowledge() messages on this subscription.
    send_empty_value: true
    properties:
      - name: 'deadLetterTopic'
        type: String
        description: |
          The name of the topic to which dead letter messages should be published.
          Format is `projects/{project}/topics/{topic}`.

          The Cloud Pub/Sub service account associated with the enclosing subscription's
          parent project (i.e.,
          service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have
          permission to Publish() to this topic.

          The operation will fail if the topic does not exist.
          Users should ensure that there is a subscription attached to this topic
          since messages published to a topic with no subscriptions are lost.
      - name: 'maxDeliveryAttempts'
        type: Integer
        description: |
          The maximum number of delivery attempts for any message. The value must be
          between 5 and 100.

          The number of delivery attempts is defined as 1 + (the sum of number of
          NACKs and number of times the acknowledgement deadline has been exceeded for the message).

          A NACK is any call to ModifyAckDeadline with a 0 deadline. Note that
          client libraries may automatically extend ack_deadlines.

          This field will be honored on a best effort basis.

          If this parameter is 0, a default value of 5 is used.
  - name: 'retryPolicy'
    type: NestedObject
    description: |
      A policy that specifies how Pub/Sub retries message delivery for this subscription.

      If not set, the default retry policy is applied. This generally implies that messages will be retried as soon as possible for healthy subscribers.
      RetryPolicy will be triggered on NACKs or acknowledgement deadline exceeded events for a given message
    properties:
      - name: 'minimumBackoff'
        type: String
        description: |
          The minimum delay between consecutive deliveries of a given message. Value should be between 0 and 600 seconds. Defaults to 10 seconds.
          A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s".
        default_from_api: true
        diff_suppress_func: 'tpgresource.DurationDiffSuppress'
      - name: 'maximumBackoff'
        type: String
        description: |
          The maximum delay between consecutive deliveries of a given message. Value should be between 0 and 600 seconds. Defaults to 600 seconds.
          A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s".
        default_from_api: true
        diff_suppress_func: 'tpgresource.DurationDiffSuppress'
    send_empty_value: true
    allow_empty_object: true
  - name: 'enableMessageOrdering'
    type: Boolean
    description: |
      If `true`, messages published with the same orderingKey in PubsubMessage will be delivered to
      the subscribers in the order in which they are received by the Pub/Sub system. Otherwise, they
      may be delivered in any order.
    immutable: true
  - name: 'enableExactlyOnceDelivery'
    type: Boolean
    description: |
      If `true`, Pub/Sub provides the following guarantees for the delivery
      of a message with a given value of messageId on this Subscriptions':

      - The message sent to a subscriber is guaranteed not to be resent before the message's acknowledgement deadline expires.

      - An acknowledged message will not be resent to a subscriber.

      Note that subscribers may still receive multiple copies of a message when `enable_exactly_once_delivery`
      is true if the message was published multiple times by a publisher client. These copies are considered distinct by Pub/Sub and have distinct messageId values