Skip to content

Commit 221ed45

Browse files
jamesiarmesjamesiarmes
authored
feat: Added exporter ephemeral service. (#31)
1 parent 9a3a4b6 commit 221ed45

File tree

15 files changed

+235
-35
lines changed

15 files changed

+235
-35
lines changed

‎.github/actions/setup-opentofu/action.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@ runs:
2424
run: tofu version
2525
- name: Set optional variables
2626
shell: bash
27+
env:
28+
TF_VAR_REGION: ${{ env.AWS_REGION }}
29+
# For any of the defined variables that have a value set into TF_VAR_*
30+
# (all uppercase), we set the corresponding TF_VAR_* (lowercase) variable
31+
# that OpenTofu expects.
2732
run: |
2833
variables=(
2934
"apply_database_updates_immediately" "consumer_container_count"
3035
"consumer_cpu" "consumer_memory" "database_instance_count"
3136
"database_skip_final_snapshot" "deletion_protection"
3237
"deployment_environments" "environment" "export_expiration"
33-
"image_tags_mutable" "key_recovery_period" "program" "project" "repository"
38+
"image_tags_mutable" "key_recovery_period" "log_level" "program"
39+
"project" "region" "repository"
3440
)
3541
for var in ${variables[@]}; do
3642
name="TF_VAR_$(echo $var | tr '[:lower:]' '[:upper:]')"

‎.github/workflows/deploy.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ jobs:
5151
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
5252
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
5353
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
54+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
5455
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
5556
TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
5657
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
@@ -65,6 +66,7 @@ jobs:
6566
needs: plan
6667
environment: ${{ inputs.environment || 'development' }}
6768
env:
69+
AWS_REGION: ${{ secrets.AWS_REGION }}
6870
TF_VAR_image_tag: ${{ inputs.image_tag || github.sha }}
6971
# Set required variables.
7072
TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
@@ -100,6 +102,7 @@ jobs:
100102
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
101103
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
102104
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
105+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
103106
TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
104107
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
105108
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

‎.github/workflows/export.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Trigger an export from Senzing to S3
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
environment:
7+
description: Environment to run the exporter in.
8+
default: development
9+
required: true
10+
type: environment
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
jobs:
17+
launch:
18+
name: Trigger export in ${{ inputs.environment }}
19+
runs-on: ubuntu-latest
20+
environment: ${{ inputs.environment }}
21+
env:
22+
# Set required variables.
23+
TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
24+
TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }}
25+
TF_VAR_vpc_private_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }}
26+
TF_VAR_vpc_public_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }}
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
- name: Set up AWS credentials
31+
uses: aws-actions/configure-aws-credentials@v4
32+
with:
33+
aws-region: ${{ secrets.AWS_REGION || 'us-west-1' }}
34+
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
35+
role-session-name: GitHub_to_AWS_via_FederatedOIDC
36+
- name: Setup OpenTofu
37+
uses: ./.github/actions/setup-opentofu
38+
env:
39+
TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
40+
TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
41+
TF_VAR_CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
42+
TF_VAR_CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
43+
TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
44+
TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }}
45+
TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
46+
TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
47+
TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }}
48+
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
49+
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
50+
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
51+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
52+
TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
53+
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
54+
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
55+
TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}
56+
TF_VAR_VPC_CIDR: ${{ secrets.TF_VAR_VPC_CIDR }}
57+
TF_VAR_VPC_PRIVATE_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }}
58+
TF_VAR_VPC_PUBLIC_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }}
59+
with:
60+
config: service
61+
- name: Get OpenTofu outputs
62+
id: outputs
63+
working-directory: ./tofu/config/service
64+
run: |
65+
OUTPUTS=$(tofu output -json | jq -c)
66+
echo "OUTPUTS=$OUTPUTS"
67+
echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT
68+
- name: Parse subnets
69+
id: subnets
70+
env:
71+
SUBNETS: ${{ toJson(fromJson(steps.outputs.outputs.outputs).container_subnets.value) }}
72+
run: |
73+
SUBNET_STRING=$(echo "$SUBNETS" | jq -r '.[]')
74+
echo "subnets<<EOF" >> $GITHUB_OUTPUT
75+
echo "$SUBNET_STRING" >> $GITHUB_OUTPUT
76+
echo "EOF" >> $GITHUB_OUTPUT
77+
- name: Launch container
78+
id: run-task
79+
uses: geekcell/github-action-aws-ecs-run-task@v5
80+
with:
81+
cluster: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}
82+
task-definition: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter
83+
override-container: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter
84+
assign-public-ip: DISABLED
85+
tail-logs: true
86+
task-wait-until-stopped: true
87+
# The block style indicator (|) is necessary to tell YAML to preserve
88+
# newlines.
89+
subnet-ids: |
90+
${{ steps.subnets.outputs.subnets }}
91+
security-group-ids: |
92+
${{ fromJson(steps.outputs.outputs.outputs).task_security_group_id.value }}

‎.github/workflows/launch-tools.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
workflow_dispatch:
55
inputs:
66
environment:
7-
description: Environment to destroy.
7+
description: Environment to launch into.
88
default: development
99
required: true
1010
type: environment
@@ -26,6 +26,7 @@ jobs:
2626
runs-on: ubuntu-latest
2727
environment: ${{ inputs.environment }}
2828
env:
29+
AWS_REGION: ${{ secrets.AWS_REGION }}
2930
# Set required variables.
3031
TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
3132
TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }}
@@ -51,10 +52,11 @@ jobs:
5152
TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }}
5253
TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
5354
TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
54-
TF_VAR_ENVIRONMENT: ${{ inputs.environment }}
55+
TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }}
5556
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
5657
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
5758
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
59+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
5860
TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
5961
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
6062
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

‎.github/workflows/plan.yaml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,14 @@ on:
1818
required: false
1919
type: string
2020
secrets:
21+
# Required secrets.
2122
AWS_REGION:
2223
AWS_ROLE_ARN:
24+
TF_VAR_REPO_OIDC_ARN:
25+
TF_VAR_VPC_CIDR:
26+
TF_VAR_VPC_PRIVATE_SUBNET_CIDRS:
27+
TF_VAR_VPC_PUBLIC_SUBNET_CIDRS:
28+
# Optional secrets.
2329
TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY:
2430
required: false
2531
TF_VAR_CONSUMER_CONTAINER_COUNT:
@@ -42,16 +48,14 @@ on:
4248
required: false
4349
TF_VAR_KEY_RECOVERY_PERIOD:
4450
required: false
51+
TF_VAR_LOG_LEVEL:
52+
required: false
4553
TF_VAR_PROGRAM:
4654
required: false
4755
TF_VAR_PROJECT:
4856
required: false
49-
TF_VAR_REPO_OIDC_ARN:
5057
TF_VAR_REPOSITORY:
5158
required: false
52-
TF_VAR_VPC_CIDR:
53-
TF_VAR_VPC_PRIVATE_SUBNET_CIDRS:
54-
TF_VAR_VPC_PUBLIC_SUBNET_CIDRS:
5559
workflow_dispatch:
5660
inputs:
5761
config:
@@ -113,6 +117,7 @@ jobs:
113117
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
114118
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
115119
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
120+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
116121
TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
117122
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
118123
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

‎.github/workflows/pull-request.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ jobs:
7575
TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
7676
TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
7777
TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
78+
TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
7879
TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
7980
TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
8081
TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}

‎.trivyignore.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
misconfigurations:
2-
# The tools Dockerfile doesn't need a healthcheck.
2+
# Dockerfiles for ephemeral containers don't need a healthcheck.
33
- id: AVD-DS-0026
44
paths:
5-
- Dockerfile.exporter # ephemeral container, healthcheck not necessary
5+
- Dockerfile.exporter
66
- Dockerfile.tools

‎Dockerfile.exporter‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,8 @@ ENV PYTHONPATH=/opt/senzing/er/sdk/python:/app
1919
# Flush buffer - helps with print statements.
2020
ENV PYTHONUNBUFFERED=1
2121

22+
# Define volumes necessary to support a read-only root filesystem on ECS
23+
# Fargate.
24+
VOLUME ["/home/senzing", "/var/lib/amazon", "/var/log"]
25+
2226
CMD ["python3", "exporter.py"]

‎tofu/config/service/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ module "system" {
3535
deletion_protection = var.deletion_protection
3636
image_tag = local.image_tag
3737
image_tags_mutable = var.image_tags_mutable
38+
log_level = var.log_level
3839

3940
consumer_container_count = var.consumer_container_count
4041
consumer_cpu = var.consumer_cpu

‎tofu/config/service/variables.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,17 @@ variable "key_recovery_period" {
8080
}
8181
}
8282

83+
variable "log_level" {
84+
type = string
85+
description = "Log level for all containers."
86+
default = "INFO"
87+
88+
validation {
89+
condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.log_level)
90+
error_message = "Valid log levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL."
91+
}
92+
}
93+
8394
variable "program" {
8495
type = string
8596
description = "Program the application belongs to."

0 commit comments

Comments
 (0)