feat: per module requirements configs for pubsub

Makefile

@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.23
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.25
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
@@ -80,7 +80,7 @@ docker_generate_docs:
 		-e ENABLE_BPMETADATA='1' \
 		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
-		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs display'
+		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs --display --per-module-requirements'
 
 # Alias for backwards compatibility
 .PHONY: generate_docs

metadata.yaml

@@ -65,7 +65,26 @@ spec:
         defaultValue: {}
       - name: push_subscriptions
         description: The list of the push subscriptions.
-        varType: "list(object({\r\n    name                       = string,\r\n    ack_deadline_seconds       = optional(number),\r\n    push_endpoint              = optional(string),\r\n    x-goog-version             = optional(string),\r\n    oidc_service_account_email = optional(string),\r\n    audience                   = optional(string),\r\n    expiration_policy          = optional(string),\r\n    dead_letter_topic          = optional(string),\r\n    retain_acked_messages      = optional(bool),\r\n    message_retention_duration = optional(string),\r\n    max_delivery_attempts      = optional(number),\r\n    maximum_backoff            = optional(string),\r\n    minimum_backoff            = optional(string),\r\n    filter                     = optional(string),\r\n    enable_message_ordering    = optional(bool),\r\n    no_wrapper                 = optional(bool),\r\n    write_metadata             = optional(bool),\r\n  }))"
+        varType: |-
+          list(object({
+              name                       = string,
+              ack_deadline_seconds       = optional(number),
+              push_endpoint              = optional(string),
+              x-goog-version             = optional(string),
+              oidc_service_account_email = optional(string),
+              audience                   = optional(string),
+              expiration_policy          = optional(string),
+              dead_letter_topic          = optional(string),
+              retain_acked_messages      = optional(bool),
+              message_retention_duration = optional(string),
+              max_delivery_attempts      = optional(number),
+              maximum_backoff            = optional(string),
+              minimum_backoff            = optional(string),
+              filter                     = optional(string),
+              enable_message_ordering    = optional(bool),
+              no_wrapper                 = optional(bool),
+              write_metadata             = optional(bool),
+            }))
         defaultValue: []
         connections:
           - source:
@@ -75,7 +94,22 @@ spec:
               outputExpr: "{ \"name\": apphub_service_uri.service_id, \"push_endpoint\": service_uri, \"oidc_service_account_email\": service_account_id.email }"
       - name: pull_subscriptions
         description: The list of the pull subscriptions.
-        varType: "list(object({\r\n    name                         = string,\r\n    ack_deadline_seconds         = optional(number),\r\n    expiration_policy            = optional(string),\r\n    dead_letter_topic            = optional(string),\r\n    max_delivery_attempts        = optional(number),\r\n    retain_acked_messages        = optional(bool),\r\n    message_retention_duration   = optional(string),\r\n    maximum_backoff              = optional(string),\r\n    minimum_backoff              = optional(string),\r\n    filter                       = optional(string),\r\n    enable_message_ordering      = optional(bool),\r\n    service_account              = optional(string),\r\n    enable_exactly_once_delivery = optional(bool),\r\n  }))"
+        varType: |-
+          list(object({
+              name                         = string,
+              ack_deadline_seconds         = optional(number),
+              expiration_policy            = optional(string),
+              dead_letter_topic            = optional(string),
+              max_delivery_attempts        = optional(number),
+              retain_acked_messages        = optional(bool),
+              message_retention_duration   = optional(string),
+              maximum_backoff              = optional(string),
+              minimum_backoff              = optional(string),
+              filter                       = optional(string),
+              enable_message_ordering      = optional(bool),
+              service_account              = optional(string),
+              enable_exactly_once_delivery = optional(bool),
+            }))
         defaultValue: []
         connections:
           - source:
@@ -90,7 +124,25 @@ spec:
               outputExpr: "{ \"name\": account_details.id, \"service_account\": account_details.email }"
       - name: bigquery_subscriptions
         description: The list of the Bigquery push subscriptions.
-        varType: "list(object({\r\n    name                       = string,\r\n    table                      = string,\r\n    use_topic_schema           = optional(bool),\r\n    use_table_schema           = optional(bool),\r\n    write_metadata             = optional(bool),\r\n    drop_unknown_fields        = optional(bool),\r\n    ack_deadline_seconds       = optional(number),\r\n    retain_acked_messages      = optional(bool),\r\n    message_retention_duration = optional(string),\r\n    enable_message_ordering    = optional(bool),\r\n    expiration_policy          = optional(string),\r\n    filter                     = optional(string),\r\n    dead_letter_topic          = optional(string),\r\n    max_delivery_attempts      = optional(number),\r\n    maximum_backoff            = optional(string),\r\n    minimum_backoff            = optional(string)\r\n  }))"
+        varType: |-
+          list(object({
+              name                       = string,
+              table                      = string,
+              use_topic_schema           = optional(bool),
+              use_table_schema           = optional(bool),
+              write_metadata             = optional(bool),
+              drop_unknown_fields        = optional(bool),
+              ack_deadline_seconds       = optional(number),
+              retain_acked_messages      = optional(bool),
+              message_retention_duration = optional(string),
+              enable_message_ordering    = optional(bool),
+              expiration_policy          = optional(string),
+              filter                     = optional(string),
+              dead_letter_topic          = optional(string),
+              max_delivery_attempts      = optional(number),
+              maximum_backoff            = optional(string),
+              minimum_backoff            = optional(string)
+            }))
         defaultValue: []
         connections:
           - source:
@@ -100,7 +152,30 @@ spec:
               outputExpr: "{ \"name\": table_ids[0], \"table\": \"${project}:${env_vars.BIGQUERY_DATASET}.${table_ids[0]}\"}"
       - name: cloud_storage_subscriptions
         description: The list of the Cloud Storage push subscriptions.
-        varType: "list(object({\r\n    name                       = string,\r\n    bucket                     = string,\r\n    filename_prefix            = optional(string),\r\n    filename_suffix            = optional(string),\r\n    filename_datetime_format   = optional(string),\r\n    max_duration               = optional(string),\r\n    max_bytes                  = optional(string),\r\n    max_messages               = optional(string),\r\n    output_format              = optional(string),\r\n    write_metadata             = optional(bool),\r\n    use_topic_schema           = optional(bool),\r\n    ack_deadline_seconds       = optional(number),\r\n    retain_acked_messages      = optional(bool),\r\n    message_retention_duration = optional(string),\r\n    enable_message_ordering    = optional(bool),\r\n    expiration_policy          = optional(string),\r\n    filter                     = optional(string),\r\n    dead_letter_topic          = optional(string),\r\n    max_delivery_attempts      = optional(number),\r\n    maximum_backoff            = optional(string),\r\n    minimum_backoff            = optional(string)\r\n  }))"
+        varType: |-
+          list(object({
+              name                       = string,
+              bucket                     = string,
+              filename_prefix            = optional(string),
+              filename_suffix            = optional(string),
+              filename_datetime_format   = optional(string),
+              max_duration               = optional(string),
+              max_bytes                  = optional(string),
+              max_messages               = optional(string),
+              output_format              = optional(string),
+              write_metadata             = optional(bool),
+              use_topic_schema           = optional(bool),
+              ack_deadline_seconds       = optional(number),
+              retain_acked_messages      = optional(bool),
+              message_retention_duration = optional(string),
+              enable_message_ordering    = optional(bool),
+              expiration_policy          = optional(string),
+              filter                     = optional(string),
+              dead_letter_topic          = optional(string),
+              max_delivery_attempts      = optional(number),
+              maximum_backoff            = optional(string),
+              minimum_backoff            = optional(string)
+            }))
         defaultValue: []
         connections:
           - source:
@@ -136,7 +211,13 @@ spec:
         defaultValue: true
       - name: schema
         description: Schema for the topic.
-        varType: "object({\r\n    name       = string\r\n    type       = string\r\n    definition = string\r\n    encoding   = string\r\n  })"
+        varType: |-
+          object({
+              name       = string
+              type       = string
+              definition = string
+              encoding   = string
+            })
     outputs:
       - name: env_vars
         description: Map of pull subscription IDs, keyed by project_subscription name for environment variables.
@@ -176,10 +257,10 @@ spec:
           - roles/bigquery.admin
           - roles/storage.admin
     services:
+      - bigquery.googleapis.com
       - cloudresourcemanager.googleapis.com
       - pubsub.googleapis.com
       - serviceusage.googleapis.com
-      - bigquery.googleapis.com
       - storage.googleapis.com
     providerVersions:
       - source: hashicorp/google

test/setup/iam.tf

@@ -15,12 +15,16 @@
  */
 
 locals {
-  int_required_roles = [
-    "roles/pubsub.admin",
-    "roles/resourcemanager.projectIamAdmin",
-    "roles/bigquery.admin",
-    "roles/storage.admin"
-  ]
+  per_module_roles = {
+    root = [
+      "roles/pubsub.admin",
+      "roles/resourcemanager.projectIamAdmin",
+      "roles/bigquery.admin",
+      "roles/storage.admin"
+    ]
+  }
+
+  int_required_roles = tolist(toset(flatten(values(local.per_module_roles))))
 }
 
 resource "random_id" "random_suffix" {

test/setup/main.tf

@@ -14,6 +14,18 @@
  * limitations under the License.
  */
 
+locals {
+  per_module_services = {
+    root = [
+      "cloudresourcemanager.googleapis.com",
+      "pubsub.googleapis.com",
+      "serviceusage.googleapis.com",
+      "bigquery.googleapis.com",
+      "storage.googleapis.com"
+    ]
+  }
+}
+
 module "project-ci-int-pubsub" {
   source  = "terraform-google-modules/project-factory/google"
   version = "~> 18.0"
@@ -24,11 +36,5 @@ module "project-ci-int-pubsub" {
   folder_id         = var.folder_id
   billing_account   = var.billing_account
 
-  activate_apis = [
-    "cloudresourcemanager.googleapis.com",
-    "pubsub.googleapis.com",
-    "serviceusage.googleapis.com",
-    "bigquery.googleapis.com",
-    "storage.googleapis.com"
-  ]
+  activate_apis = tolist(toset(flatten(values(local.per_module_services))))
 }