From c9501c4bb8f7877a1b4a363d2c6d814b58c9c4f0 Mon Sep 17 00:00:00 2001 From: khaledt Date: Fri, 24 Jan 2025 18:47:04 +0000 Subject: [PATCH 01/13] chore: add output type metada --- metadata.display.yaml | 6 +++++- metadata.yaml | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/metadata.display.yaml b/metadata.display.yaml index 84e91ef1..470c6b55 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -22,7 +22,7 @@ spec: info: title: terraform-google-pubsub source: - repo: https://github.com/ktinubu/terraform-google-pubsub.git + repo: https://github.com/terraform-google-modules/terraform-google-pubsub sourceType: git ui: input: @@ -75,3 +75,7 @@ spec: topic_message_retention_duration: name: topic_message_retention_duration title: Topic Message Retention Duration + runtime: + outputs: + service_uri: + visibility: VISIBILITY_ROOT diff --git a/metadata.yaml b/metadata.yaml index f666b3f3..f62cacad 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -113,16 +113,25 @@ spec: outputs: - name: id description: The ID of the Pub/Sub topic + type: string - name: subscription_names description: The name list of Pub/Sub subscriptions + type: + - tuple + - [] - name: subscription_paths description: The path list of Pub/Sub subscriptions + type: + - tuple + - [] - name: topic description: The name of the Pub/Sub topic + type: string - name: topic_labels description: Labels assigned to the Pub/Sub topic - name: uri description: The URI of the Pub/Sub topic + type: string requirements: roles: - level: Project From 1e581d9f76496e116dfcedf64df47477342fd651 Mon Sep 17 00:00:00 2001 From: khaledt Date: Fri, 24 Jan 2025 18:50:37 +0000 Subject: [PATCH 02/13] enable metadata for lint generate docs docker commands --- Makefile | 2 ++ examples/cloud_storage/main.tf | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ba8f87b8..958c7d10 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ docker_test_integration: .PHONY: docker_test_lint docker_test_lint: docker run --rm -it \ + -e ENABLE_BPMETADATA='1' \ -v "$(CURDIR)":/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /usr/local/bin/test_lint.sh @@ -76,6 +77,7 @@ docker_test_lint: .PHONY: docker_generate_docs docker_generate_docs: docker run --rm -it \ + -e ENABLE_BPMETADATA='1' \ -v "$(CURDIR)":/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs' diff --git a/examples/cloud_storage/main.tf b/examples/cloud_storage/main.tf index 0a7ec2dd..0225d7d2 100644 --- a/examples/cloud_storage/main.tf +++ b/examples/cloud_storage/main.tf @@ -47,8 +47,8 @@ module "pubsub" { } resource "google_storage_bucket" "test" { - project = var.project_id - name = join("-", ["test_bucket", random_id.bucket_suffix.hex]) - location = "europe-west1" + project = var.project_id + name = join("-", ["test_bucket", random_id.bucket_suffix.hex]) + location = "europe-west1" uniform_bucket_level_access = true } From b671424dbd40798d69fa582e2f8785aad4b07713 Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 28 Jan 2025 15:44:52 +0000 Subject: [PATCH 03/13] chore: add connection metadata --- metadata.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/metadata.yaml b/metadata.yaml index f62cacad..1e41a538 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -75,10 +75,24 @@ spec: description: The list of the Bigquery push subscriptions. varType: list(map(string)) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules//terraform-google-bigquery + version: ">= 9.0.0" + spec: + outputExpr: table_ids[0] + inputPath: ["table"] - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. varType: list(map(string)) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules//terraform-google-cloud-storage + version: ">= 9.0.1" + spec: + inputPath: ["bucket"] + outputExpr: name - name: subscription_labels description: A map of labels to assign to every Pub/Sub subscription. varType: map(string) @@ -93,6 +107,13 @@ spec: - name: topic_kms_key_name description: The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. varType: string + # connections: + # - source: + # source: github.com/terraform-google-modules//terraform-google-kms + # version: ">= 3.2.0" + # spec: + # outputExpr: keys(keys)[0] + # inputPath: topic_kms_key_name - name: grant_bigquery_project_roles description: Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. varType: bool From e43ada3a2f0fe9747b137261edb4e16099e7113f Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 28 Jan 2025 17:19:30 +0000 Subject: [PATCH 04/13] fix connections, change bq sub to list(object)) --- README.md | 2 +- metadata.yaml | 36 ++++++++++++++++++++++-------------- variables.tf | 9 ++++++++- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e5333f17..3d25b583 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | `list(map(string))` | `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
              name : string,
              table : string,
              use_topic_schema : optional(bool, true),
              use_table_schema : optional(bool, false),
              write_metadata : optional(bool, false),
              drop_unknown_fields : optional(bool, false)
            }))
| `[]` | no | | cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | `list(map(string))` | `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | diff --git a/metadata.yaml b/metadata.yaml index 1e41a538..0ec81b4d 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -73,26 +73,34 @@ spec: defaultValue: [] - name: bigquery_subscriptions description: The list of the Bigquery push subscriptions. - varType: list(map(string)) + varType: |- + list(object({ + name : string, + table : string, + use_topic_schema : optional(bool, true), + use_table_schema : optional(bool, false), + write_metadata : optional(bool, false), + drop_unknown_fields : optional(bool, false) + })) defaultValue: [] - connections: + connections: - source: - source: github.com/terraform-google-modules//terraform-google-bigquery + source: github.com/terraform-google-modules/terraform-google-bigquery version: ">= 9.0.0" spec: outputExpr: table_ids[0] - inputPath: ["table"] + inputPath: table - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. varType: list(map(string)) defaultValue: [] - connections: + connections: - source: - source: github.com/terraform-google-modules//terraform-google-cloud-storage + source: github.com/terraform-google-modules/terraform-google-cloud-storage version: ">= 9.0.1" spec: - inputPath: ["bucket"] outputExpr: name + inputPath: bucket - name: subscription_labels description: A map of labels to assign to every Pub/Sub subscription. varType: map(string) @@ -107,13 +115,13 @@ spec: - name: topic_kms_key_name description: The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. varType: string - # connections: - # - source: - # source: github.com/terraform-google-modules//terraform-google-kms - # version: ">= 3.2.0" - # spec: - # outputExpr: keys(keys)[0] - # inputPath: topic_kms_key_name + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-kms + version: ">= 3.2.0" + spec: + outputExpr: keys(keys)[0] + inputPath: topic_kms_key_name - name: grant_bigquery_project_roles description: Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. varType: bool diff --git a/variables.tf b/variables.tf index a1f62775..a8de23cf 100644 --- a/variables.tf +++ b/variables.tf @@ -54,7 +54,14 @@ variable "pull_subscriptions" { } variable "bigquery_subscriptions" { - type = list(map(string)) + type = list(object({ + name : string, + table : string, + use_topic_schema : optional(bool, true), + use_table_schema : optional(bool, false), + write_metadata : optional(bool, false), + drop_unknown_fields : optional(bool, false) + })) description = "The list of the Bigquery push subscriptions." default = [] } From c429a994bf70c14501a8a14e5556a25bbdba328b Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 28 Jan 2025 19:37:04 +0000 Subject: [PATCH 05/13] fix --- README.md | 4 ++-- metadata.yaml | 31 ++++++++++++++++++++++--------- variables.tf | 14 +++++++++++++- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3d25b583..bb7c5c1b 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
              name : string,
              table : string,
              use_topic_schema : optional(bool, true),
              use_table_schema : optional(bool, false),
              write_metadata : optional(bool, false),
              drop_unknown_fields : optional(bool, false)
            }))
| `[]` | no | -| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | `list(map(string))` | `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false)
  }))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
  }))
| `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/metadata.yaml b/metadata.yaml index 0ec81b4d..637430ba 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -75,13 +75,13 @@ spec: description: The list of the Bigquery push subscriptions. varType: |- list(object({ - name : string, - table : string, - use_topic_schema : optional(bool, true), - use_table_schema : optional(bool, false), - write_metadata : optional(bool, false), - drop_unknown_fields : optional(bool, false) - })) + name : string, + table : string, + use_topic_schema : optional(bool, true), + use_table_schema : optional(bool, false), + write_metadata : optional(bool, false), + drop_unknown_fields : optional(bool, false) + })) defaultValue: [] connections: - source: @@ -92,11 +92,24 @@ spec: inputPath: table - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. - varType: list(map(string)) + varType: |- + list(object({ + name : string, + bucket : string, + filename_prefix : optional(string), + filename_suffix : optional(string), + filename_datetime_format : optional(string), + max_duration : optional(string), + max_bytes : optional(string), + max_messages : optional(string), + output_format : optional(string), + write_metadata : optional(bool, false), + use_topic_schema : optional(bool, false), + })) defaultValue: [] connections: - source: - source: github.com/terraform-google-modules/terraform-google-cloud-storage + source: github.com/terraform-google-modules/terraform-google-cloud-storage//modules/simple_bucket version: ">= 9.0.1" spec: outputExpr: name diff --git a/variables.tf b/variables.tf index a8de23cf..2c23e594 100644 --- a/variables.tf +++ b/variables.tf @@ -67,7 +67,19 @@ variable "bigquery_subscriptions" { } variable "cloud_storage_subscriptions" { - type = list(map(string)) + type = list(object({ + name : string, + bucket : string, + filename_prefix : optional(string), + filename_suffix : optional(string), + filename_datetime_format : optional(string), + max_duration : optional(string), + max_bytes : optional(string), + max_messages : optional(string), + output_format : optional(string), + write_metadata : optional(bool, false), + use_topic_schema : optional(bool, false), + })) description = "The list of the Cloud Storage push subscriptions." default = [] } From d41828a55e8182ccc6900e24f83acffe0da56f55 Mon Sep 17 00:00:00 2001 From: khaledt Date: Wed, 29 Jan 2025 16:09:59 +0000 Subject: [PATCH 06/13] fix: add generic subscription configs to bq and cloud storage sub inputs --- README.md | 4 ++-- metadata.yaml | 18 ++++++++++++++++++ variables.tf | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb7c5c1b..f6b35726 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false)
  }))
| `[]` | no | -| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
  }))
| `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false)
    ack_deadline_seconds : optional(number)
    retain_acked_messages : optional(bool, false)
    message_retention_duration : optional(string)
    enable_message_ordering : optional(bool, false)
    expiration_policy : optional(string)
    filter : optional(string)
    dead_letter_topic : optional(string)
    maximum_backoff : optional(string)
    minimum_backoff : optional(string)
  }))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
    ack_deadline_seconds : optional(number)
    retain_acked_messages : optional(bool, false)
    message_retention_duration : optional(string)
    enable_message_ordering : optional(bool, false)
    expiration_policy : optional(string)
    filter : optional(string)
    dead_letter_topic : optional(string)
    maximum_backoff : optional(string)
    minimum_backoff : optional(string)
  }))
| `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/metadata.yaml b/metadata.yaml index 637430ba..ed241658 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -81,6 +81,15 @@ spec: use_table_schema : optional(bool, false), write_metadata : optional(bool, false), drop_unknown_fields : optional(bool, false) + ack_deadline_seconds : optional(number) + retain_acked_messages : optional(bool, false) + message_retention_duration : optional(string) + enable_message_ordering : optional(bool, false) + expiration_policy : optional(string) + filter : optional(string) + dead_letter_topic : optional(string) + maximum_backoff : optional(string) + minimum_backoff : optional(string) })) defaultValue: [] connections: @@ -105,6 +114,15 @@ spec: output_format : optional(string), write_metadata : optional(bool, false), use_topic_schema : optional(bool, false), + ack_deadline_seconds : optional(number) + retain_acked_messages : optional(bool, false) + message_retention_duration : optional(string) + enable_message_ordering : optional(bool, false) + expiration_policy : optional(string) + filter : optional(string) + dead_letter_topic : optional(string) + maximum_backoff : optional(string) + minimum_backoff : optional(string) })) defaultValue: [] connections: diff --git a/variables.tf b/variables.tf index 2c23e594..b6b3978a 100644 --- a/variables.tf +++ b/variables.tf @@ -61,6 +61,15 @@ variable "bigquery_subscriptions" { use_table_schema : optional(bool, false), write_metadata : optional(bool, false), drop_unknown_fields : optional(bool, false) + ack_deadline_seconds : optional(number) + retain_acked_messages : optional(bool, false) + message_retention_duration : optional(string) + enable_message_ordering : optional(bool, false) + expiration_policy : optional(string) + filter : optional(string) + dead_letter_topic : optional(string) + maximum_backoff : optional(string) + minimum_backoff : optional(string) })) description = "The list of the Bigquery push subscriptions." default = [] @@ -79,6 +88,15 @@ variable "cloud_storage_subscriptions" { output_format : optional(string), write_metadata : optional(bool, false), use_topic_schema : optional(bool, false), + ack_deadline_seconds : optional(number) + retain_acked_messages : optional(bool, false) + message_retention_duration : optional(string) + enable_message_ordering : optional(bool, false) + expiration_policy : optional(string) + filter : optional(string) + dead_letter_topic : optional(string) + maximum_backoff : optional(string) + minimum_backoff : optional(string) })) description = "The list of the Cloud Storage push subscriptions." default = [] From 91ef36759f8d80ecab2918364075f3a7fa3174e4 Mon Sep 17 00:00:00 2001 From: khaledt Date: Wed, 29 Jan 2025 19:27:55 +0000 Subject: [PATCH 07/13] fix --- README.md | 4 ++-- main.tf | 6 +++--- metadata.yaml | 34 +++++++++++++++++----------------- variables.tf | 34 +++++++++++++++++----------------- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index f6b35726..cd90c33c 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false)
    ack_deadline_seconds : optional(number)
    retain_acked_messages : optional(bool, false)
    message_retention_duration : optional(string)
    enable_message_ordering : optional(bool, false)
    expiration_policy : optional(string)
    filter : optional(string)
    dead_letter_topic : optional(string)
    maximum_backoff : optional(string)
    minimum_backoff : optional(string)
  }))
| `[]` | no | -| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
    ack_deadline_seconds : optional(number)
    retain_acked_messages : optional(bool, false)
    message_retention_duration : optional(string)
    enable_message_ordering : optional(bool, false)
    expiration_policy : optional(string)
    filter : optional(string)
    dead_letter_topic : optional(string)
    maximum_backoff : optional(string)
    minimum_backoff : optional(string)
  }))
| `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false),
    ack_deadline_seconds : optional(number),
    retain_acked_messages : optional(bool, false),
    message_retention_duration : optional(string),
    enable_message_ordering : optional(bool, false),
    expiration_policy : optional(string),
    filter : optional(string),
    dead_letter_topic : optional(string),
    maximum_backoff : optional(string),
    minimum_backoff : optional(string)
  }))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
    ack_deadline_seconds : optional(number),
    retain_acked_messages : optional(bool, false),
    message_retention_duration : optional(string),
    enable_message_ordering : optional(bool, false),
    expiration_policy : optional(string),
    filter : optional(string),
    dead_letter_topic : optional(string),
    maximum_backoff : optional(string),
    minimum_backoff : optional(string)
  }))
| `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/main.tf b/main.tf index 45755f7f..1edc22ef 100644 --- a/main.tf +++ b/main.tf @@ -345,14 +345,14 @@ resource "google_pubsub_subscription" "bigquery_subscriptions" { ) dynamic "expiration_policy" { // check if the 'expiration_policy' key exists, if yes, return a list containing it. - for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : [] + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] content { ttl = expiration_policy.value } } dynamic "dead_letter_policy" { - for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : [] + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] content { dead_letter_topic = lookup(each.value, "dead_letter_topic", "") max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5") @@ -416,7 +416,7 @@ resource "google_pubsub_subscription" "cloud_storage_subscriptions" { ) dynamic "expiration_policy" { // check if the 'expiration_policy' key exists, if yes, return a list containing it. - for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : [] + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] content { ttl = expiration_policy.value } diff --git a/metadata.yaml b/metadata.yaml index ed241658..688942b3 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -80,15 +80,15 @@ spec: use_topic_schema : optional(bool, true), use_table_schema : optional(bool, false), write_metadata : optional(bool, false), - drop_unknown_fields : optional(bool, false) - ack_deadline_seconds : optional(number) - retain_acked_messages : optional(bool, false) - message_retention_duration : optional(string) - enable_message_ordering : optional(bool, false) - expiration_policy : optional(string) - filter : optional(string) - dead_letter_topic : optional(string) - maximum_backoff : optional(string) + drop_unknown_fields : optional(bool, false), + ack_deadline_seconds : optional(number), + retain_acked_messages : optional(bool, false), + message_retention_duration : optional(string), + enable_message_ordering : optional(bool, false), + expiration_policy : optional(string), + filter : optional(string), + dead_letter_topic : optional(string), + maximum_backoff : optional(string), minimum_backoff : optional(string) })) defaultValue: [] @@ -114,14 +114,14 @@ spec: output_format : optional(string), write_metadata : optional(bool, false), use_topic_schema : optional(bool, false), - ack_deadline_seconds : optional(number) - retain_acked_messages : optional(bool, false) - message_retention_duration : optional(string) - enable_message_ordering : optional(bool, false) - expiration_policy : optional(string) - filter : optional(string) - dead_letter_topic : optional(string) - maximum_backoff : optional(string) + ack_deadline_seconds : optional(number), + retain_acked_messages : optional(bool, false), + message_retention_duration : optional(string), + enable_message_ordering : optional(bool, false), + expiration_policy : optional(string), + filter : optional(string), + dead_letter_topic : optional(string), + maximum_backoff : optional(string), minimum_backoff : optional(string) })) defaultValue: [] diff --git a/variables.tf b/variables.tf index b6b3978a..86f66199 100644 --- a/variables.tf +++ b/variables.tf @@ -60,15 +60,15 @@ variable "bigquery_subscriptions" { use_topic_schema : optional(bool, true), use_table_schema : optional(bool, false), write_metadata : optional(bool, false), - drop_unknown_fields : optional(bool, false) - ack_deadline_seconds : optional(number) - retain_acked_messages : optional(bool, false) - message_retention_duration : optional(string) - enable_message_ordering : optional(bool, false) - expiration_policy : optional(string) - filter : optional(string) - dead_letter_topic : optional(string) - maximum_backoff : optional(string) + drop_unknown_fields : optional(bool, false), + ack_deadline_seconds : optional(number), + retain_acked_messages : optional(bool, false), + message_retention_duration : optional(string), + enable_message_ordering : optional(bool, false), + expiration_policy : optional(string), + filter : optional(string), + dead_letter_topic : optional(string), + maximum_backoff : optional(string), minimum_backoff : optional(string) })) description = "The list of the Bigquery push subscriptions." @@ -88,14 +88,14 @@ variable "cloud_storage_subscriptions" { output_format : optional(string), write_metadata : optional(bool, false), use_topic_schema : optional(bool, false), - ack_deadline_seconds : optional(number) - retain_acked_messages : optional(bool, false) - message_retention_duration : optional(string) - enable_message_ordering : optional(bool, false) - expiration_policy : optional(string) - filter : optional(string) - dead_letter_topic : optional(string) - maximum_backoff : optional(string) + ack_deadline_seconds : optional(number), + retain_acked_messages : optional(bool, false), + message_retention_duration : optional(string), + enable_message_ordering : optional(bool, false), + expiration_policy : optional(string), + filter : optional(string), + dead_letter_topic : optional(string), + maximum_backoff : optional(string), minimum_backoff : optional(string) })) description = "The list of the Cloud Storage push subscriptions." From ca24340a7d6da2a60b0f23de4097b7e2825aa950 Mon Sep 17 00:00:00 2001 From: khaledt Date: Wed, 29 Jan 2025 20:04:55 +0000 Subject: [PATCH 08/13] fix --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 1edc22ef..57da59b7 100644 --- a/main.tf +++ b/main.tf @@ -87,7 +87,7 @@ resource "google_pubsub_topic_iam_member" "pull_topic_binding" { } resource "google_pubsub_topic_iam_member" "bigquery_topic_binding" { - for_each = var.create_topic ? { for i in var.bigquery_subscriptions : i.name => i if try(i.dead_letter_topic, "") != "" } : {} + for_each = var.create_topic ? { for i in var.bigquery_subscriptions : i.name => i if i.dead_letter_topic != null } : {} project = var.project_id topic = each.value.dead_letter_topic @@ -423,7 +423,7 @@ resource "google_pubsub_subscription" "cloud_storage_subscriptions" { } dynamic "dead_letter_policy" { - for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : [] + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] content { dead_letter_topic = lookup(each.value, "dead_letter_topic", "") max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5") From b5ec3260f17cc24d886c10718ebda09e5d1820fc Mon Sep 17 00:00:00 2001 From: khaledt Date: Fri, 31 Jan 2025 21:03:19 +0000 Subject: [PATCH 09/13] use correct name format for bq table and kms key --- metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index 688942b3..5450d7fa 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -97,7 +97,7 @@ spec: source: github.com/terraform-google-modules/terraform-google-bigquery version: ">= 9.0.0" spec: - outputExpr: table_ids[0] + outputExpr: values(bigquery_external_tables)[0].id inputPath: table - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. @@ -151,7 +151,7 @@ spec: source: github.com/terraform-google-modules/terraform-google-kms version: ">= 3.2.0" spec: - outputExpr: keys(keys)[0] + outputExpr: values(keys)[0].id inputPath: topic_kms_key_name - name: grant_bigquery_project_roles description: Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. From b83bab99716bbdf1e09fc217c7b4940270c3d816 Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 4 Feb 2025 13:40:21 +0000 Subject: [PATCH 10/13] use external_table_ids for bq table connection --- metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.yaml b/metadata.yaml index 5450d7fa..e09e977b 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -97,7 +97,7 @@ spec: source: github.com/terraform-google-modules/terraform-google-bigquery version: ">= 9.0.0" spec: - outputExpr: values(bigquery_external_tables)[0].id + outputExpr: external_table_ids[0] inputPath: table - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. From 7dd9b360dd72a1959529ee52caa7b83561f69ec7 Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 4 Feb 2025 13:44:28 +0000 Subject: [PATCH 11/13] remove topic_kms_key_name connection --- metadata.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index e09e977b..ad6327f2 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -146,13 +146,6 @@ spec: - name: topic_kms_key_name description: The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. varType: string - connections: - - source: - source: github.com/terraform-google-modules/terraform-google-kms - version: ">= 3.2.0" - spec: - outputExpr: values(keys)[0].id - inputPath: topic_kms_key_name - name: grant_bigquery_project_roles description: Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. varType: bool From 65e6ed24b9b245bc613b6f18401af1e34017f650 Mon Sep 17 00:00:00 2001 From: khaledt Date: Tue, 4 Feb 2025 13:58:28 +0000 Subject: [PATCH 12/13] Require bigquery version 10.0.0 for connection metadata --- metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.yaml b/metadata.yaml index ad6327f2..f7806780 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -95,7 +95,7 @@ spec: connections: - source: source: github.com/terraform-google-modules/terraform-google-bigquery - version: ">= 9.0.0" + version: ">= 10.0.0" spec: outputExpr: external_table_ids[0] inputPath: table From 409a54b91a2def810b9fd0758aa02919e7020054 Mon Sep 17 00:00:00 2001 From: khaledt Date: Fri, 7 Feb 2025 00:27:34 +0000 Subject: [PATCH 13/13] use `=` instead of `:` for object variables --- README.md | 4 +-- metadata.yaml | 70 +++++++++++++++++++++++++-------------------------- variables.tf | 70 +++++++++++++++++++++++++-------------------------- 3 files changed, 72 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index cd90c33c..23b2b44b 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ module "pubsub" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name : string,
    table : string,
    use_topic_schema : optional(bool, true),
    use_table_schema : optional(bool, false),
    write_metadata : optional(bool, false),
    drop_unknown_fields : optional(bool, false),
    ack_deadline_seconds : optional(number),
    retain_acked_messages : optional(bool, false),
    message_retention_duration : optional(string),
    enable_message_ordering : optional(bool, false),
    expiration_policy : optional(string),
    filter : optional(string),
    dead_letter_topic : optional(string),
    maximum_backoff : optional(string),
    minimum_backoff : optional(string)
  }))
| `[]` | no | -| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name : string,
    bucket : string,
    filename_prefix : optional(string),
    filename_suffix : optional(string),
    filename_datetime_format : optional(string),
    max_duration : optional(string),
    max_bytes : optional(string),
    max_messages : optional(string),
    output_format : optional(string),
    write_metadata : optional(bool, false),
    use_topic_schema : optional(bool, false),
    ack_deadline_seconds : optional(number),
    retain_acked_messages : optional(bool, false),
    message_retention_duration : optional(string),
    enable_message_ordering : optional(bool, false),
    expiration_policy : optional(string),
    filter : optional(string),
    dead_letter_topic : optional(string),
    maximum_backoff : optional(string),
    minimum_backoff : optional(string)
  }))
| `[]` | no | +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. | 
list(object({
    name                       = string,
    table                      = string,
    use_topic_schema           = optional(bool, true),
    use_table_schema           = optional(bool, false),
    write_metadata             = optional(bool, false),
    drop_unknown_fields        = optional(bool, false),
    ack_deadline_seconds       = optional(number),
    retain_acked_messages      = optional(bool, false),
    message_retention_duration = optional(string),
    enable_message_ordering    = optional(bool, false),
    expiration_policy          = optional(string),
    filter                     = optional(string),
    dead_letter_topic          = optional(string),
    maximum_backoff            = optional(string),
    minimum_backoff            = optional(string)
  }))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. | 
list(object({
    name                       = string,
    bucket                     = string,
    filename_prefix            = optional(string),
    filename_suffix            = optional(string),
    filename_datetime_format   = optional(string),
    max_duration               = optional(string),
    max_bytes                  = optional(string),
    max_messages               = optional(string),
    output_format              = optional(string),
    write_metadata             = optional(bool, false),
    use_topic_schema           = optional(bool, false),
    ack_deadline_seconds       = optional(number),
    retain_acked_messages      = optional(bool, false),
    message_retention_duration = optional(string),
    enable_message_ordering    = optional(bool, false),
    expiration_policy          = optional(string),
    filter                     = optional(string),
    dead_letter_topic          = optional(string),
    maximum_backoff            = optional(string),
    minimum_backoff            = optional(string)
  }))
| `[]` | no | | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no | | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no | | grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | diff --git a/metadata.yaml b/metadata.yaml index f7806780..a017e464 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -75,21 +75,21 @@ spec: description: The list of the Bigquery push subscriptions. varType: |- list(object({ - name : string, - table : string, - use_topic_schema : optional(bool, true), - use_table_schema : optional(bool, false), - write_metadata : optional(bool, false), - drop_unknown_fields : optional(bool, false), - ack_deadline_seconds : optional(number), - retain_acked_messages : optional(bool, false), - message_retention_duration : optional(string), - enable_message_ordering : optional(bool, false), - expiration_policy : optional(string), - filter : optional(string), - dead_letter_topic : optional(string), - maximum_backoff : optional(string), - minimum_backoff : optional(string) + name = string, + table = string, + use_topic_schema = optional(bool, true), + use_table_schema = optional(bool, false), + write_metadata = optional(bool, false), + drop_unknown_fields = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) })) defaultValue: [] connections: @@ -103,26 +103,26 @@ spec: description: The list of the Cloud Storage push subscriptions. varType: |- list(object({ - name : string, - bucket : string, - filename_prefix : optional(string), - filename_suffix : optional(string), - filename_datetime_format : optional(string), - max_duration : optional(string), - max_bytes : optional(string), - max_messages : optional(string), - output_format : optional(string), - write_metadata : optional(bool, false), - use_topic_schema : optional(bool, false), - ack_deadline_seconds : optional(number), - retain_acked_messages : optional(bool, false), - message_retention_duration : optional(string), - enable_message_ordering : optional(bool, false), - expiration_policy : optional(string), - filter : optional(string), - dead_letter_topic : optional(string), - maximum_backoff : optional(string), - minimum_backoff : optional(string) + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool, false), + use_topic_schema = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) })) defaultValue: [] connections: diff --git a/variables.tf b/variables.tf index 86f66199..643c82a6 100644 --- a/variables.tf +++ b/variables.tf @@ -55,21 +55,21 @@ variable "pull_subscriptions" { variable "bigquery_subscriptions" { type = list(object({ - name : string, - table : string, - use_topic_schema : optional(bool, true), - use_table_schema : optional(bool, false), - write_metadata : optional(bool, false), - drop_unknown_fields : optional(bool, false), - ack_deadline_seconds : optional(number), - retain_acked_messages : optional(bool, false), - message_retention_duration : optional(string), - enable_message_ordering : optional(bool, false), - expiration_policy : optional(string), - filter : optional(string), - dead_letter_topic : optional(string), - maximum_backoff : optional(string), - minimum_backoff : optional(string) + name = string, + table = string, + use_topic_schema = optional(bool, true), + use_table_schema = optional(bool, false), + write_metadata = optional(bool, false), + drop_unknown_fields = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) })) description = "The list of the Bigquery push subscriptions." default = [] @@ -77,26 +77,26 @@ variable "bigquery_subscriptions" { variable "cloud_storage_subscriptions" { type = list(object({ - name : string, - bucket : string, - filename_prefix : optional(string), - filename_suffix : optional(string), - filename_datetime_format : optional(string), - max_duration : optional(string), - max_bytes : optional(string), - max_messages : optional(string), - output_format : optional(string), - write_metadata : optional(bool, false), - use_topic_schema : optional(bool, false), - ack_deadline_seconds : optional(number), - retain_acked_messages : optional(bool, false), - message_retention_duration : optional(string), - enable_message_ordering : optional(bool, false), - expiration_policy : optional(string), - filter : optional(string), - dead_letter_topic : optional(string), - maximum_backoff : optional(string), - minimum_backoff : optional(string) + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool, false), + use_topic_schema = optional(bool, false), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool, false), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool, false), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) })) description = "The list of the Cloud Storage push subscriptions." default = []