diff --git a/examples/push_subscription-separate-pub-sub/README.md b/examples/push_subscription-separate-pub-sub/README.md new file mode 100644 index 00000000..52e33662 --- /dev/null +++ b/examples/push_subscription-separate-pub-sub/README.md @@ -0,0 +1,35 @@ +# Simple Example + +This example illustrates how to use the `pub` and `sub` module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| project\_id | The project ID | +| topic\_labels | The labels of the Pub/Sub topic created | +| topic\_name | The name of the Pub/Sub topic created | + + + +## Requirements + +The following sections describe the requirements which must be met in +order to invoke this example. The requirements of the +[root module][root-module-requirements] must be met. + +## Usage + +To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within +this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/push_subscription-separate-pub-sub/main.tf b/examples/push_subscription-separate-pub-sub/main.tf new file mode 100644 index 00000000..8711b378 --- /dev/null +++ b/examples/push_subscription-separate-pub-sub/main.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "pub" { + source = "terraform-google-modules/pubsub/google//modules/pub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic-cr-push" + topic_labels = { + foo_label = "foo_value" + bar_label = "bar_value" + } +} + +module "sub" { + source = "terraform-google-modules/pubsub/google//modules/sub" + version = "~> 7.0" + + project_id = var.project_id + topic = module.pub.topic + + push_subscriptions = [ + { + name = module.cloud-run.service_name + push_endpoint = module.cloud-run.service_uri + oidc_service_account_email = module.cloud-run.service_account_id.email + }, + ] +} + +module "cloud-run" { + source = "GoogleCloudPlatform/cloud-run/google//modules/v2" + version = "~> 0.17" + project_id = var.project_id + location = "us-central1" + service_name = "cr-service" + containers = [{ "container_name" = "", "container_image" = "gcr.io/design-center-container-repo/pubsub-cr-push:latest-1703" }] + service_account_project_roles = ["roles/run.invoker"] + members = ["allUsers"] + cloud_run_deletion_protection = false +} diff --git a/examples/push_subscription-separate-pub-sub/outputs.tf b/examples/push_subscription-separate-pub-sub/outputs.tf new file mode 100644 index 00000000..0d72909f --- /dev/null +++ b/examples/push_subscription-separate-pub-sub/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id + description = "The project ID" +} + +output "topic_name" { + value = module.pub.topic + description = "The name of the Pub/Sub topic created" +} + +output "topic_labels" { + value = module.pub.topic_labels + description = "The labels of the Pub/Sub topic created" +} diff --git a/examples/push_subscription-separate-pub-sub/variables.tf b/examples/push_subscription-separate-pub-sub/variables.tf new file mode 100644 index 00000000..5abd8d87 --- /dev/null +++ b/examples/push_subscription-separate-pub-sub/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources" +} diff --git a/metadata.yaml b/metadata.yaml index 5121ae47..2a587a7e 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -46,6 +46,8 @@ spec: location: examples/cloud_storage-separate-pub-sub - name: kms location: examples/kms + - name: push_subscription-separate-pub-sub + location: examples/push_subscription-separate-pub-sub - name: simple location: examples/simple - name: simple-separate-pub-sub @@ -166,12 +168,19 @@ spec: - roles/resourcemanager.projectIamAdmin - roles/bigquery.admin - roles/storage.admin + - roles/run.admin + - roles/iam.serviceAccountAdmin + - roles/iam.serviceAccountUser + - roles/resourcemanager.projectIamAdmin + - roles/logging.viewer services: - cloudresourcemanager.googleapis.com - pubsub.googleapis.com - serviceusage.googleapis.com - bigquery.googleapis.com - storage.googleapis.com + - run.googleapis.com + - iam.googleapis.com providerVersions: - source: hashicorp/google version: ">= 6.2, < 7" diff --git a/modules/pub/README.md b/modules/pub/README.md index 0eff07dc..62b33c16 100644 --- a/modules/pub/README.md +++ b/modules/pub/README.md @@ -41,6 +41,7 @@ module "pub" { |------|-------------|------|---------|:--------:| | message\_storage\_policy | A map of storage policies. Default - inherit from organization's Resource Location Restriction policy. | `map(any)` | `{}` | no | | project\_id | The project ID to manage the Pub/Sub resources. | `string` | n/a | yes | +| publisher\_service\_accounts | Service account email which required roles/pubsub.publisher role. | 
list(object({
    id              = string
    service_account = string
  }))
| `[]` | no | | schema | Schema for the topic. | 
object({
    name       = string
    type       = string
    definition = string
    encoding   = string
  })
| `null` | no | | topic | The Pub/Sub topic name. | `string` | n/a | yes | | topic\_kms\_key\_name | The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. | `string` | `null` | no | diff --git a/modules/pub/main.tf b/modules/pub/main.tf index 5f565cbf..167a62d8 100644 --- a/modules/pub/main.tf +++ b/modules/pub/main.tf @@ -45,3 +45,12 @@ resource "google_pubsub_topic" "topic" { } depends_on = [google_pubsub_schema.schema] } + +resource "google_pubsub_topic_iam_member" "sa_binding_publisher" { + for_each = { for i in var.publisher_service_accounts : i.id => i if i.service_account != null } + + project = var.project_id + topic = var.topic + role = "roles/pubsub.publisher" + member = "serviceAccount:${each.value.service_account}" +} diff --git a/modules/pub/metadata.display.yaml b/modules/pub/metadata.display.yaml index 9277a821..90d11eef 100644 --- a/modules/pub/metadata.display.yaml +++ b/modules/pub/metadata.display.yaml @@ -37,6 +37,9 @@ spec: project_id: name: project_id title: Project Id + publisher_service_accounts: + name: publisher_service_accounts + title: Publisher Service Accounts schema: name: schema title: Schema diff --git a/modules/pub/metadata.yaml b/modules/pub/metadata.yaml index 12786c92..d76537a5 100644 --- a/modules/pub/metadata.yaml +++ b/modules/pub/metadata.yaml @@ -42,6 +42,8 @@ spec: location: examples/cloud_storage-separate-pub-sub - name: kms location: examples/kms + - name: push_subscription-separate-pub-sub + location: examples/push_subscription-separate-pub-sub - name: simple location: examples/simple - name: simple-separate-pub-sub @@ -81,6 +83,25 @@ spec: definition = string encoding = string }) + - name: publisher_service_accounts + description: Service account email which required roles/pubsub.publisher role. + varType: |- + list(object({ + id = string + service_account = string + })) + defaultValue: [] + connections: + - source: + source: github.com/GoogleCloudPlatform/terraform-google-cloud-run//modules/v2 + version: ">= 0.13" + spec: + outputExpr: "{ \"id\": service_account_id.id, \"service_account\": service_account_id.email }" + - source: + source: github.com/terraform-google-modules/terraform-google-service-accounts//modules/simple-sa + version: ">= 4.4" + spec: + outputExpr: "{ \"id\": account_details.id, \"service_account\": account_details.email }" outputs: - name: id description: The ID of the Pub/Sub topic @@ -98,12 +119,19 @@ spec: - roles/resourcemanager.projectIamAdmin - roles/bigquery.admin - roles/storage.admin + - roles/run.admin + - roles/iam.serviceAccountAdmin + - roles/iam.serviceAccountUser + - roles/resourcemanager.projectIamAdmin + - roles/logging.viewer services: - cloudresourcemanager.googleapis.com - pubsub.googleapis.com - serviceusage.googleapis.com - bigquery.googleapis.com - storage.googleapis.com + - run.googleapis.com + - iam.googleapis.com providerVersions: - source: hashicorp/google version: ">= 6.2, < 7" diff --git a/modules/pub/variables.tf b/modules/pub/variables.tf index b9d50da6..32874a14 100644 --- a/modules/pub/variables.tf +++ b/modules/pub/variables.tf @@ -58,3 +58,12 @@ variable "schema" { description = "Schema for the topic." default = null } + +variable "publisher_service_accounts" { + type = list(object({ + id = string + service_account = string + })) + description = "Service account email which required roles/pubsub.publisher role." + default = [] +} diff --git a/modules/sub/README.md b/modules/sub/README.md index da842577..e57c4471 100644 --- a/modules/sub/README.md +++ b/modules/sub/README.md @@ -71,6 +71,7 @@ module "sub" { | Name | Description | |------|-------------| +| pull\_subscription\_env\_vars | Map of pull subscription IDs, keyed by project\_subscription name for environment variables. | | subscription\_names | The name list of Pub/Sub subscriptions | | subscription\_paths | The path list of Pub/Sub subscriptions | diff --git a/modules/sub/metadata.yaml b/modules/sub/metadata.yaml index 443c879c..1da356c0 100644 --- a/modules/sub/metadata.yaml +++ b/modules/sub/metadata.yaml @@ -42,6 +42,8 @@ spec: location: examples/cloud_storage-separate-pub-sub - name: kms location: examples/kms + - name: push_subscription-separate-pub-sub + location: examples/push_subscription-separate-pub-sub - name: simple location: examples/simple - name: simple-separate-pub-sub @@ -58,6 +60,12 @@ spec: description: The Pub/Sub topic name. varType: string required: true + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-pubsub//modules/pub + version: ">= 7.0.0" + spec: + outputExpr: topic - name: push_subscriptions description: The list of the push subscriptions. varType: |- @@ -79,6 +87,12 @@ spec: enable_message_ordering = optional(bool), })) defaultValue: [] + connections: + - source: + source: github.com/GoogleCloudPlatform/terraform-google-cloud-run//modules/v2 + version: ">= 0.13" + spec: + outputExpr: "{ \"name\": apphub_service_uri.service_id, \"push_endpoint\": service_uri, \"oidc_service_account_email\": service_account_id.email }" - name: pull_subscriptions description: The list of the pull subscriptions. varType: |- @@ -98,6 +112,17 @@ spec: enable_exactly_once_delivery = optional(bool), })) defaultValue: [] + connections: + - source: + source: github.com/GoogleCloudPlatform/terraform-google-cloud-run//modules/v2 + version: ">= 0.13" + spec: + outputExpr: "{ \"name\": service_name, \"service_account\": service_account_id.email }" + - source: + source: github.com/terraform-google-modules/terraform-google-service-accounts//modules/simple-sa + version: ">= 4.4" + spec: + outputExpr: "{ \"name\": account_details.id, \"service_account\": account_details.email }" - name: bigquery_subscriptions description: The list of the Bigquery push subscriptions. varType: |- @@ -119,6 +144,12 @@ spec: minimum_backoff = optional(string) })) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-bigquery + version: ">= 10.0.0" + spec: + outputExpr: "{ \"name\": external_table_ids[0], \"table\": external_table_ids[0]}" - name: cloud_storage_subscriptions description: The list of the Cloud Storage push subscriptions. varType: |- @@ -145,6 +176,12 @@ spec: minimum_backoff = optional(string) })) defaultValue: [] + connections: + - source: + source: github.com/terraform-google-modules/terraform-google-cloud-storage//modules/simple_bucket + version: ">= 9.0.1" + spec: + outputExpr: "{ \"name\": name, \"bucket\": name}" - name: subscription_labels description: A map of labels to assign to every Pub/Sub subscription. varType: map(string) @@ -158,6 +195,8 @@ spec: varType: bool defaultValue: true outputs: + - name: pull_subscription_env_vars + description: Map of pull subscription IDs, keyed by project_subscription name for environment variables. - name: subscription_names description: The name list of Pub/Sub subscriptions - name: subscription_paths @@ -170,12 +209,19 @@ spec: - roles/resourcemanager.projectIamAdmin - roles/bigquery.admin - roles/storage.admin + - roles/run.admin + - roles/iam.serviceAccountAdmin + - roles/iam.serviceAccountUser + - roles/resourcemanager.projectIamAdmin + - roles/logging.viewer services: - cloudresourcemanager.googleapis.com - pubsub.googleapis.com - serviceusage.googleapis.com - bigquery.googleapis.com - storage.googleapis.com + - run.googleapis.com + - iam.googleapis.com providerVersions: - source: hashicorp/google version: ">= 6.2, < 7" diff --git a/modules/sub/outputs.tf b/modules/sub/outputs.tf index f214870f..81354b64 100644 --- a/modules/sub/outputs.tf +++ b/modules/sub/outputs.tf @@ -36,3 +36,10 @@ output "subscription_paths" { description = "The path list of Pub/Sub subscriptions" } +output "pull_subscription_env_vars" { + value = { + for k, v in google_pubsub_subscription.pull_subscriptions : replace(upper("${v.project}_${v.name}"), "-", "_") => v.id + } + + description = "Map of pull subscription IDs, keyed by project_subscription name for environment variables." +} diff --git a/test/integration/push_subscription-separate-pub-sub/push_subscription_separate_pub_sub_test.go b/test/integration/push_subscription-separate-pub-sub/push_subscription_separate_pub_sub_test.go new file mode 100755 index 00000000..3428a099 --- /dev/null +++ b/test/integration/push_subscription-separate-pub-sub/push_subscription_separate_pub_sub_test.go @@ -0,0 +1,60 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package push_subscription_separate_pub_sub + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestPushSubscriptionSeparatePubSub(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + projectId := bpt.GetStringOutput("project_id") + + op := gcloud.Runf(t, "pubsub topics describe cft-tf-pub-topic-cr-push --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-cr-push", projectId), op.Get("name").String(), "has expected name") + assert.Equal("bar_value", op.Get("labels.bar_label").String(), "has expected labels") + assert.Equal("foo_value", op.Get("labels.foo_label").String(), "has expected labels") + + op = gcloud.Runf(t, "pubsub subscriptions describe cr-service --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/subscriptions/cr-service", projectId), op.Get("name").String(), "has expected name") + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-cr-push", projectId), op.Get("topic").String(), "has expected topic") + // Publish a test message + message := "Hello Runner!" + publishCmd := fmt.Sprintf("pubsub topics publish cft-tf-pub-topic-cr-push --message='%s' --project=%s", message, projectId) + gcloud.Runf(t, publishCmd) + + // Wait for a short time to allow the message to be processed. Adjust if necessary. + time.Sleep(30 * time.Second) + + // Check Cloud Run logs for the message + logCmd := fmt.Sprintf("logging read \"resource.type=cloud_run_revision AND resource.labels.service_name=cr-service AND textPayload:\\\"%s\\\"\" --project=%s --limit=1", message, projectId) + logOp := gcloud.Runf(t, logCmd) + + // Assert that the log entry is found + assert.Contains(logOp.String(), message, "Cloud Run logs should contain the published message") + }) + + bpt.Test() +} diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 5a41966c..93379a28 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -19,7 +19,12 @@ locals { "roles/pubsub.admin", "roles/resourcemanager.projectIamAdmin", "roles/bigquery.admin", - "roles/storage.admin" + "roles/storage.admin", + "roles/run.admin", + "roles/iam.serviceAccountAdmin", + "roles/iam.serviceAccountUser", + "roles/resourcemanager.projectIamAdmin", + "roles/logging.viewer" ] } diff --git a/test/setup/main.tf b/test/setup/main.tf index deaa9e15..d7069bee 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -29,6 +29,8 @@ module "project-ci-int-pubsub" { "pubsub.googleapis.com", "serviceusage.googleapis.com", "bigquery.googleapis.com", - "storage.googleapis.com" + "storage.googleapis.com", + "run.googleapis.com", + "iam.googleapis.com" ] }