Skip to content

Commit cc30fc2

Browse files
q2wq2w
committed
Add submodules for pub and sub
1 parent fcc6f01 commit cc30fc2

File tree

21 files changed

+1676
-1
lines changed

21 files changed

+1676
-1
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ docker_generate_docs:
8080
-e ENABLE_BPMETADATA='1' \
8181
-v "$(CURDIR)":/workspace \
8282
$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
83-
/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs'
83+
/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs display'
8484

8585
# Alias for backwards compatibility
8686
.PHONY: generate_docs

‎examples/simple-separate-pub-sub/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Simple Example
2+
3+
This example illustrates how to use the `pubsub` module.
4+
5+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
6+
## Inputs
7+
8+
| Name | Description | Type | Default | Required |
9+
|------|-------------|------|---------|:--------:|
10+
| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes |
11+
12+
## Outputs
13+
14+
| Name | Description |
15+
|------|-------------|
16+
| project\_id | The project ID |
17+
| topic\_labels | The labels of the Pub/Sub topic created |
18+
| topic\_name | The name of the Pub/Sub topic created |
19+
20+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
21+
22+
## Requirements
23+
24+
The following sections describe the requirements which must be met in
25+
order to invoke this example. The requirements of the
26+
[root module][root-module-requirements] must be met.
27+
28+
## Usage
29+
30+
To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within
31+
this directory:
32+
- `terraform init` to get the plugins
33+
- `terraform plan` to see the infrastructure plan
34+
- `terraform apply` to apply the infrastructure build
35+
- `terraform destroy` to destroy the built infrastructure

‎examples/simple-separate-pub-sub/main.tf‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
provider "google" {
18+
region = "us-central1"
19+
}
20+
21+
module "pub" {
22+
source = "terraform-google-modules/pubsub/google//modules/pub"
23+
version = "~> 7.0"
24+
25+
project_id = var.project_id
26+
topic = "cft-tf-pub-topic"
27+
topic_labels = {
28+
foo_label = "foo_value"
29+
bar_label = "bar_value"
30+
}
31+
32+
schema = {
33+
name = "pub-example"
34+
type = "AVRO"
35+
encoding = "JSON"
36+
definition = "{\n \"type\" : \"record\",\n \"name\" : \"Avro\",\n \"fields\" : [\n {\n \"name\" : \"StringField\",\n \"type\" : \"string\"\n },\n {\n \"name\" : \"IntField\",\n \"type\" : \"int\"\n }\n ]\n}\n"
37+
}
38+
}
39+
40+
module "sub" {
41+
source = "terraform-google-modules/pubsub/google//modules/sub"
42+
version = "~> 7.0"
43+
44+
project_id = var.project_id
45+
topic = module.pub.topic
46+
47+
pull_subscriptions = [
48+
{
49+
name = "sub-pull"
50+
ack_deadline_seconds = 10
51+
enable_exactly_once_delivery = true
52+
},
53+
{
54+
name = "sub-pull2"
55+
minimum_backoff = "10s"
56+
maximum_backoff = "600s"
57+
expiration_policy = "1209600s" // two weeks
58+
}
59+
]
60+
61+
push_subscriptions = [
62+
{
63+
name = "sub-push"
64+
push_endpoint = "https://${var.project_id}.appspot.com/"
65+
x-goog-version = "v1beta1"
66+
ack_deadline_seconds = 20
67+
expiration_policy = "1209600s" // two weeks
68+
},
69+
{
70+
name = "sub-push-default-expiration"
71+
push_endpoint = "https://${var.project_id}.appspot.com/"
72+
x-goog-version = "v1beta1"
73+
},
74+
]
75+
}

‎examples/simple-separate-pub-sub/outputs.tf‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
output "project_id" {
18+
value = var.project_id
19+
description = "The project ID"
20+
}
21+
22+
output "topic_name" {
23+
value = module.pub.topic
24+
description = "The name of the Pub/Sub topic created"
25+
}
26+
27+
output "topic_labels" {
28+
value = module.pub.topic_labels
29+
description = "The labels of the Pub/Sub topic created"
30+
}

‎examples/simple-separate-pub-sub/variables.tf‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
variable "project_id" {
18+
type = string
19+
description = "The project ID to manage the Pub/Sub resources"
20+
}

‎metadata.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ spec:
3030
version: ">= 1.3"
3131
description: {}
3232
content:
33+
subBlueprints:
34+
- name: pub
35+
location: modules/pub
36+
- name: sub
37+
location: modules/sub
3338
examples:
3439
- name: bigquery
3540
location: examples/bigquery
@@ -39,6 +44,8 @@ spec:
3944
location: examples/kms
4045
- name: simple
4146
location: examples/simple
47+
- name: simple-separate-pub-sub
48+
location: examples/simple-separate-pub-sub
4249
- name: subscriptions_only
4350
location: examples/subscriptions_only
4451
interfaces:

‎modules/pub/README.md‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# terraform-google-pubsub
2+
3+
This module makes it easy to create Google Cloud Pub/Sub topic and subscriptions associated with the topic.
4+
5+
## Compatibility
6+
This module is meant for use with Terraform 0.13+ and tested using Terraform 1.0+. If you find incompatibilities using Terraform >=0.13, please open an issue.
7+
If you haven't
8+
[upgraded](https://www.terraform.io/upgrade-guides/0-13.html) and need a Terraform
9+
0.12.x-compatible version of this module, the last released version
10+
intended for Terraform 0.12.x is [v1.9.0](https://registry.terraform.io/modules/terraform-google-modules/-pubsub/google/v1.9.0).
11+
12+
## Usage
13+
14+
This is a simple usage of the module. Please see also a simple setup provided in the example directory.
15+
16+
```hcl
17+
module "pubsub" {
18+
source = "terraform-google-modules/pubsub/google"
19+
version = "~> 7.0"
20+
21+
topic = "tf-topic"
22+
project_id = "my-pubsub-project"
23+
push_subscriptions = [
24+
{
25+
name = "push" // required
26+
ack_deadline_seconds = 20 // optional
27+
push_endpoint = "https://example.com" // required
28+
x-goog-version = "v1beta1" // optional
29+
oidc_service_account_email = "[email protected]" // optional
30+
audience = "example" // optional
31+
expiration_policy = "1209600s" // optional
32+
dead_letter_topic = "projects/my-pubsub-project/topics/example-dl-topic" // optional
33+
max_delivery_attempts = 5 // optional
34+
maximum_backoff = "600s" // optional
35+
minimum_backoff = "300s" // optional
36+
filter = "attributes.domain = \"com\"" // optional
37+
enable_message_ordering = true // optional
38+
}
39+
]
40+
pull_subscriptions = [
41+
{
42+
name = "pull" // required
43+
ack_deadline_seconds = 20 // optional
44+
dead_letter_topic = "projects/my-pubsub-project/topics/example-dl-topic" // optional
45+
max_delivery_attempts = 5 // optional
46+
maximum_backoff = "600s" // optional
47+
minimum_backoff = "300s" // optional
48+
filter = "attributes.domain = \"com\"" // optional
49+
enable_message_ordering = true // optional
50+
service_account = "[email protected]" // optional
51+
enable_exactly_once_delivery = true // optional
52+
}
53+
]
54+
bigquery_subscriptions = [
55+
{
56+
name = "bigquery" // required
57+
table = "project.dataset.table" // required
58+
use_topic_schema = true // optional
59+
use_table_schema = false // optional
60+
write_metadata = false // optional
61+
drop_unknown_fields = false // optional
62+
}
63+
]
64+
cloud_storage_subscriptions = [
65+
{
66+
name = "cloud-storage" // required
67+
bucket = "example-bucket" // required
68+
filename_prefix = "log_events_" // optional
69+
filename_suffix = ".avro" // optional
70+
filename_datetime_format = "YYYY-MM-DD/hh_mm_ssZ" // optional
71+
max_duration = "60s" // optional
72+
max_bytes = "10000000" // optional
73+
max_messages = "10000" // optional
74+
output_format = "avro" // optional
75+
write_metadata = false // optional
76+
use_topic_schema = false // optional
77+
}
78+
]
79+
}
80+
```
81+
82+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
83+
## Inputs
84+
85+
| Name | Description | Type | Default | Required |
86+
|------|-------------|------|---------|:--------:|
87+
| create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no |
88+
| message\_storage\_policy | A map of storage policies. Default - inherit from organization's Resource Location Restriction policy. | `map(any)` | `{}` | no |
89+
| project\_id | The project ID to manage the Pub/Sub resources. | `string` | n/a | yes |
90+
| schema | Schema for the topic. | <pre>object({<br> name = string<br> type = string<br> definition = string<br> encoding = string<br> })</pre> | `null` | no |
91+
| topic | The Pub/Sub topic name. | `string` | n/a | yes |
92+
| topic\_kms\_key\_name | The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. | `string` | `null` | no |
93+
| topic\_labels | A map of labels to assign to the Pub/Sub topic. | `map(string)` | `{}` | no |
94+
| topic\_message\_retention\_duration | The minimum duration in seconds to retain a message after it is published to the topic. | `string` | `null` | no |
95+
96+
## Outputs
97+
98+
| Name | Description |
99+
|------|-------------|
100+
| id | The ID of the Pub/Sub topic |
101+
| topic | The name of the Pub/Sub topic |
102+
| topic\_labels | Labels assigned to the Pub/Sub topic |
103+
| uri | The URI of the Pub/Sub topic |
104+
105+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
106+
107+
## Requirements
108+
109+
### Installation Dependencies
110+
111+
- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0
112+
- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin >= v2.13
113+
114+
### Configure a Service Account
115+
116+
In order to execute this module you must have a Service Account with the following:
117+
118+
#### Roles
119+
120+
- `roles/pubsub.admin`
121+
122+
### Enable APIs
123+
124+
In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
125+
126+
- Cloud Pub/Sub API
127+
128+
#### Service Account Credentials
129+
130+
You can pass the service account credentials into this module by setting the following environment variables:
131+
132+
* `GOOGLE_CREDENTIALS`
133+
* `GOOGLE_CLOUD_KEYFILE_JSON`
134+
* `GCLOUD_KEYFILE_JSON`
135+
136+
See more [details](https://www.terraform.io/docs/providers/google/provider_reference.html#configuration-reference).
137+
138+
[v0.2.0]: https://registry.terraform.io/modules/terraform-google-modules/pubsub/google/0.2.0
139+
[terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html

‎modules/pub/main.tf‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
resource "google_pubsub_schema" "schema" {
18+
count = var.schema != null ? 1 : 0
19+
project = var.project_id
20+
name = var.schema.name
21+
type = var.schema.type
22+
definition = var.schema.definition
23+
}
24+
25+
resource "google_pubsub_topic" "topic" {
26+
count = var.create_topic ? 1 : 0
27+
project = var.project_id
28+
name = var.topic
29+
labels = var.topic_labels
30+
kms_key_name = var.topic_kms_key_name
31+
message_retention_duration = var.topic_message_retention_duration
32+
33+
dynamic "message_storage_policy" {
34+
for_each = var.message_storage_policy
35+
content {
36+
allowed_persistence_regions = message_storage_policy.key == "allowed_persistence_regions" ? message_storage_policy.value : null
37+
}
38+
}
39+
40+
dynamic "schema_settings" {
41+
for_each = var.schema != null ? [var.schema] : []
42+
content {
43+
schema = google_pubsub_schema.schema[0].id
44+
encoding = lookup(schema_settings.value, "encoding", null)
45+
}
46+
}
47+
depends_on = [google_pubsub_schema.schema]
48+
}

0 commit comments

Comments
 (0)