diff --git a/Makefile b/Makefile index 02f7b3ed..d1b55ee8 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.23 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.25 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd @@ -80,7 +80,7 @@ docker_generate_docs: -e ENABLE_BPMETADATA='1' \ -v "$(CURDIR)":/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ - /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs display' + /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs --display --per-module-requirements' # Alias for backwards compatibility .PHONY: generate_docs diff --git a/metadata.yaml b/metadata.yaml index d9bf1392..d90149c3 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -65,7 +65,26 @@ spec: defaultValue: {} - name: push_subscriptions description: The list of the push subscriptions. - varType: "list(object({\r\n name = string,\r\n ack_deadline_seconds = optional(number),\r\n push_endpoint = optional(string),\r\n x-goog-version = optional(string),\r\n oidc_service_account_email = optional(string),\r\n audience = optional(string),\r\n expiration_policy = optional(string),\r\n dead_letter_topic = optional(string),\r\n retain_acked_messages = optional(bool),\r\n message_retention_duration = optional(string),\r\n max_delivery_attempts = optional(number),\r\n maximum_backoff = optional(string),\r\n minimum_backoff = optional(string),\r\n filter = optional(string),\r\n enable_message_ordering = optional(bool),\r\n no_wrapper = optional(bool),\r\n write_metadata = optional(bool),\r\n }))" + varType: |- + list(object({ + name = string, + ack_deadline_seconds = optional(number), + push_endpoint = optional(string), + x-goog-version = optional(string), + oidc_service_account_email = optional(string), + audience = optional(string), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + max_delivery_attempts = optional(number), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + no_wrapper = optional(bool), + write_metadata = optional(bool), + })) defaultValue: [] connections: - source: @@ -75,7 +94,22 @@ spec: outputExpr: "{ \"name\": apphub_service_uri.service_id, \"push_endpoint\": service_uri, \"oidc_service_account_email\": service_account_id.email }" - name: pull_subscriptions description: The list of the pull subscriptions. - varType: "list(object({\r\n name = string,\r\n ack_deadline_seconds = optional(number),\r\n expiration_policy = optional(string),\r\n dead_letter_topic = optional(string),\r\n max_delivery_attempts = optional(number),\r\n retain_acked_messages = optional(bool),\r\n message_retention_duration = optional(string),\r\n maximum_backoff = optional(string),\r\n minimum_backoff = optional(string),\r\n filter = optional(string),\r\n enable_message_ordering = optional(bool),\r\n service_account = optional(string),\r\n enable_exactly_once_delivery = optional(bool),\r\n }))" + varType: |- + list(object({ + name = string, + ack_deadline_seconds = optional(number), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + max_delivery_attempts = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + service_account = optional(string), + enable_exactly_once_delivery = optional(bool), + })) defaultValue: [] connections: - source: @@ -90,7 +124,25 @@ spec: outputExpr: "{ \"name\": account_details.id, \"service_account\": account_details.email }" - name: bigquery_subscriptions description: The list of the Bigquery push subscriptions. - varType: "list(object({\r\n name = string,\r\n table = string,\r\n use_topic_schema = optional(bool),\r\n use_table_schema = optional(bool),\r\n write_metadata = optional(bool),\r\n drop_unknown_fields = optional(bool),\r\n ack_deadline_seconds = optional(number),\r\n retain_acked_messages = optional(bool),\r\n message_retention_duration = optional(string),\r\n enable_message_ordering = optional(bool),\r\n expiration_policy = optional(string),\r\n filter = optional(string),\r\n dead_letter_topic = optional(string),\r\n max_delivery_attempts = optional(number),\r\n maximum_backoff = optional(string),\r\n minimum_backoff = optional(string)\r\n }))" + varType: |- + list(object({ + name = string, + table = string, + use_topic_schema = optional(bool), + use_table_schema = optional(bool), + write_metadata = optional(bool), + drop_unknown_fields = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + max_delivery_attempts = optional(number), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) defaultValue: [] connections: - source: @@ -100,7 +152,30 @@ spec: outputExpr: "{ \"name\": table_ids[0], \"table\": \"${project}:${env_vars.BIGQUERY_DATASET}.${table_ids[0]}\"}" - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. - varType: "list(object({\r\n name = string,\r\n bucket = string,\r\n filename_prefix = optional(string),\r\n filename_suffix = optional(string),\r\n filename_datetime_format = optional(string),\r\n max_duration = optional(string),\r\n max_bytes = optional(string),\r\n max_messages = optional(string),\r\n output_format = optional(string),\r\n write_metadata = optional(bool),\r\n use_topic_schema = optional(bool),\r\n ack_deadline_seconds = optional(number),\r\n retain_acked_messages = optional(bool),\r\n message_retention_duration = optional(string),\r\n enable_message_ordering = optional(bool),\r\n expiration_policy = optional(string),\r\n filter = optional(string),\r\n dead_letter_topic = optional(string),\r\n max_delivery_attempts = optional(number),\r\n maximum_backoff = optional(string),\r\n minimum_backoff = optional(string)\r\n }))" + varType: |- + list(object({ + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool), + use_topic_schema = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + max_delivery_attempts = optional(number), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) defaultValue: [] connections: - source: @@ -136,7 +211,13 @@ spec: defaultValue: true - name: schema description: Schema for the topic. - varType: "object({\r\n name = string\r\n type = string\r\n definition = string\r\n encoding = string\r\n })" + varType: |- + object({ + name = string + type = string + definition = string + encoding = string + }) outputs: - name: env_vars description: Map of pull subscription IDs, keyed by project_subscription name for environment variables. @@ -176,10 +257,10 @@ spec: - roles/bigquery.admin - roles/storage.admin services: + - bigquery.googleapis.com - cloudresourcemanager.googleapis.com - pubsub.googleapis.com - serviceusage.googleapis.com - - bigquery.googleapis.com - storage.googleapis.com providerVersions: - source: hashicorp/google diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 5a41966c..b454d362 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -15,12 +15,16 @@ */ locals { - int_required_roles = [ - "roles/pubsub.admin", - "roles/resourcemanager.projectIamAdmin", - "roles/bigquery.admin", - "roles/storage.admin" - ] + per_module_roles = { + root = [ + "roles/pubsub.admin", + "roles/resourcemanager.projectIamAdmin", + "roles/bigquery.admin", + "roles/storage.admin" + ] + } + + int_required_roles = tolist(toset(flatten(values(local.per_module_roles)))) } resource "random_id" "random_suffix" { diff --git a/test/setup/main.tf b/test/setup/main.tf index deaa9e15..f2109454 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -14,6 +14,18 @@ * limitations under the License. */ +locals { + per_module_services = { + root = [ + "cloudresourcemanager.googleapis.com", + "pubsub.googleapis.com", + "serviceusage.googleapis.com", + "bigquery.googleapis.com", + "storage.googleapis.com" + ] + } +} + module "project-ci-int-pubsub" { source = "terraform-google-modules/project-factory/google" version = "~> 18.0" @@ -24,11 +36,5 @@ module "project-ci-int-pubsub" { folder_id = var.folder_id billing_account = var.billing_account - activate_apis = [ - "cloudresourcemanager.googleapis.com", - "pubsub.googleapis.com", - "serviceusage.googleapis.com", - "bigquery.googleapis.com", - "storage.googleapis.com" - ] + activate_apis = tolist(toset(flatten(values(local.per_module_services)))) }