diff --git a/README.md b/README.md index e5333f17..23b2b44b 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | `list(map(string))` | `[]` | no | -| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | `list(map(string))` | `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. |
list(object({
name = string,
table = string,
use_topic_schema = optional(bool, true),
use_table_schema = optional(bool, false),
write_metadata = optional(bool, false),
drop_unknown_fields = optional(bool, false),
ack_deadline_seconds = optional(number),
retain_acked_messages = optional(bool, false),
message_retention_duration = optional(string),
enable_message_ordering = optional(bool, false),
expiration_policy = optional(string),
filter = optional(string),
dead_letter_topic = optional(string),
maximum_backoff = optional(string),
minimum_backoff = optional(string)
}))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. |
list(object({
name = string,
bucket = string,
filename_prefix = optional(string),
filename_suffix = optional(string),
filename_datetime_format = optional(string),
max_duration = optional(string),
max_bytes = optional(string),
max_messages = optional(string),
output_format = optional(string),
write_metadata = optional(bool, false),
use_topic_schema = optional(bool, false),
ack_deadline_seconds = optional(number),
retain_acked_messages = optional(bool, false),
message_retention_duration = optional(string),
enable_message_ordering = optional(bool, false),
expiration_policy = optional(string),
filter = optional(string),
dead_letter_topic = optional(string),
maximum_backoff = optional(string),
minimum_backoff = optional(string)
}))
| `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/main.tf b/main.tf index 45755f7f..57da59b7 100644 --- a/main.tf +++ b/main.tf @@ -87,7 +87,7 @@ resource "google_pubsub_topic_iam_member" "pull_topic_binding" { } resource "google_pubsub_topic_iam_member" "bigquery_topic_binding" { - for_each = var.create_topic ? { for i in var.bigquery_subscriptions : i.name => i if try(i.dead_letter_topic, "") != "" } : {} + for_each = var.create_topic ? { for i in var.bigquery_subscriptions : i.name => i if i.dead_letter_topic != null } : {} project = var.project_id topic = each.value.dead_letter_topic @@ -345,14 +345,14 @@ resource "google_pubsub_subscription" "bigquery_subscriptions" { ) dynamic "expiration_policy" { // check if the 'expiration_policy' key exists, if yes, return a list containing it. - for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : [] + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] content { ttl = expiration_policy.value } } dynamic "dead_letter_policy" { - for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : [] + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] content { dead_letter_topic = lookup(each.value, "dead_letter_topic", "") max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5") @@ -416,14 +416,14 @@ resource "google_pubsub_subscription" "cloud_storage_subscriptions" { ) dynamic "expiration_policy" { // check if the 'expiration_policy' key exists, if yes, return a list containing it. - for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : [] + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] content { ttl = expiration_policy.value } } dynamic "dead_letter_policy" { - for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : [] + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] content { dead_letter_topic = lookup(each.value, "dead_letter_topic", "") max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5") diff --git a/metadata.yaml b/metadata.yaml index f62cacad..a017e464 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -73,12 +73,65 @@ spec: defaultValue: [] - name: bigquery_subscriptions description: The list of the Bigquery push subscriptions. - varType: list(map(string)) + varType: |- + list(object({ + name = string, + table = string, + use_topic_schema = optional(bool, true), + use_table_schema = optional(bool, false), + write_metadata = optional(bool, false), + drop_unknown_fields = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-bigquery + version: ">= 10.0.0" + spec: + outputExpr: external_table_ids[0] + inputPath: table - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. - varType: list(map(string)) + varType: |- + list(object({ + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool, false), + use_topic_schema = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-cloud-storage//modules/simple_bucket + version: ">= 9.0.1" + spec: + outputExpr: name + inputPath: bucket - name: subscription_labels description: A map of labels to assign to every Pub/Sub subscription. varType: map(string) diff --git a/variables.tf b/variables.tf index a1f62775..643c82a6 100644 --- a/variables.tf +++ b/variables.tf @@ -54,13 +54,50 @@ variable "pull_subscriptions" { } variable "bigquery_subscriptions" { - type = list(map(string)) + type = list(object({ + name = string, + table = string, + use_topic_schema = optional(bool, true), + use_table_schema = optional(bool, false), + write_metadata = optional(bool, false), + drop_unknown_fields = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) description = "The list of the Bigquery push subscriptions." default = [] } variable "cloud_storage_subscriptions" { - type = list(map(string)) + type = list(object({ + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool, false), + use_topic_schema = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) description = "The list of the Cloud Storage push subscriptions." default = [] }