feat: Added exporter ephemeral service.

.github/actions/setup-opentofu/action.yaml

@@ -24,13 +24,19 @@ runs:
       run: tofu version
     - name: Set optional variables
       shell: bash
+      env:
+        TF_VAR_REGION: ${{ env.AWS_REGION }}
+      # For any of the defined variables that have a value set into TF_VAR_*
+      # (all uppercase), we set the corresponding TF_VAR_* (lowercase) variable
+      # that OpenTofu expects.
       run: |
         variables=(
           "apply_database_updates_immediately" "consumer_container_count"
           "consumer_cpu" "consumer_memory" "database_instance_count"
           "database_skip_final_snapshot" "deletion_protection"
           "deployment_environments" "environment" "export_expiration"
-          "image_tags_mutable" "key_recovery_period" "program" "project" "repository"
+          "image_tags_mutable" "key_recovery_period" "log_level" "program"
+          "project" "region" "repository"
         )
         for var in ${variables[@]}; do
           name="TF_VAR_$(echo $var | tr '[:lower:]' '[:upper:]')"

.github/workflows/deploy.yaml

@@ -51,6 +51,7 @@ jobs:
       TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
       TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
       TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+      TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
       TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
       TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
       TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
@@ -65,6 +66,7 @@ jobs:
     needs: plan
     environment: ${{ inputs.environment || 'development' }}
     env:
+      AWS_REGION: ${{ secrets.AWS_REGION }}
       TF_VAR_image_tag: ${{ inputs.image_tag || github.sha }}
       # Set required variables.
       TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
@@ -100,6 +102,7 @@ jobs:
           TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
           TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
           TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+          TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
           TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
           TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
           TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

.github/workflows/export.yaml

@@ -0,0 +1,92 @@
+name: Trigger an export from Senzing to S3
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to run the exporter in.
+        default: development
+        required: true
+        type: environment
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  launch:
+    name: Trigger export in ${{ inputs.environment }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      # Set required variables.
+      TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
+      TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }}
+      TF_VAR_vpc_private_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }}
+      TF_VAR_vpc_public_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION || 'us-west-1' }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          role-session-name: GitHub_to_AWS_via_FederatedOIDC
+      - name: Setup OpenTofu
+        uses: ./.github/actions/setup-opentofu
+        env:
+          TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
+          TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
+          TF_VAR_CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
+          TF_VAR_CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
+          TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
+          TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }}
+          TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
+          TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
+          TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }}
+          TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
+          TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
+          TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+          TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
+          TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
+          TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
+          TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
+          TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}
+          TF_VAR_VPC_CIDR: ${{ secrets.TF_VAR_VPC_CIDR }}
+          TF_VAR_VPC_PRIVATE_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }}
+          TF_VAR_VPC_PUBLIC_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }}
+        with:
+          config: service
+      - name: Get OpenTofu outputs
+        id: outputs
+        working-directory: ./tofu/config/service
+        run: |
+            OUTPUTS=$(tofu output -json | jq -c)
+            echo "OUTPUTS=$OUTPUTS"
+            echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT
+      - name: Parse subnets
+        id: subnets
+        env:
+          SUBNETS: ${{ toJson(fromJson(steps.outputs.outputs.outputs).container_subnets.value) }}
+        run: |
+          SUBNET_STRING=$(echo "$SUBNETS" | jq -r '.[]')
+          echo "subnets<<EOF" >> $GITHUB_OUTPUT
+          echo "$SUBNET_STRING" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+      - name: Launch container
+        id: run-task
+        uses: geekcell/github-action-aws-ecs-run-task@v5
+        with:
+          cluster: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}
+          task-definition: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter
+          override-container: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter
+          assign-public-ip: DISABLED
+          tail-logs: true
+          task-wait-until-stopped: true
+          # The block style indicator (|) is necessary to tell YAML to preserve
+          # newlines.
+          subnet-ids: |
+            ${{ steps.subnets.outputs.subnets }}
+          security-group-ids: |
+            ${{ fromJson(steps.outputs.outputs.outputs).task_security_group_id.value }}

.github/workflows/launch-tools.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       environment:
-        description: Environment to destroy.
+        description: Environment to launch into.
         default: development
         required: true
         type: environment
@@ -26,6 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     env:
+      AWS_REGION: ${{ secrets.AWS_REGION }}
       # Set required variables.
       TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
       TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }}
@@ -51,10 +52,11 @@ jobs:
           TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }}
           TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
           TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
-          TF_VAR_ENVIRONMENT: ${{ inputs.environment }}
+          TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }}
           TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
           TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
           TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+          TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
           TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
           TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
           TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

.github/workflows/plan.yaml

@@ -18,8 +18,14 @@ on:
         required: false
         type: string
     secrets:
+      # Required secrets.
       AWS_REGION:
       AWS_ROLE_ARN:
+      TF_VAR_REPO_OIDC_ARN:
+      TF_VAR_VPC_CIDR:
+      TF_VAR_VPC_PRIVATE_SUBNET_CIDRS:
+      TF_VAR_VPC_PUBLIC_SUBNET_CIDRS:
+      # Optional secrets.
       TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY:
         required: false
       TF_VAR_CONSUMER_CONTAINER_COUNT:
@@ -42,16 +48,14 @@ on:
         required: false
       TF_VAR_KEY_RECOVERY_PERIOD:
         required: false
+      TF_VAR_LOG_LEVEL:
+        required: false
       TF_VAR_PROGRAM:
         required: false
       TF_VAR_PROJECT:
         required: false
-      TF_VAR_REPO_OIDC_ARN:
       TF_VAR_REPOSITORY:
         required: false
-      TF_VAR_VPC_CIDR:
-      TF_VAR_VPC_PRIVATE_SUBNET_CIDRS:
-      TF_VAR_VPC_PUBLIC_SUBNET_CIDRS:
   workflow_dispatch:
     inputs:
       config:
@@ -113,6 +117,7 @@ jobs:
           TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
           TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
           TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+          TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
           TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }}
           TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
           TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}

.github/workflows/pull-request.yaml

@@ -75,6 +75,7 @@ jobs:
       TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
       TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }}
       TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }}
+      TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }}
       TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }}
       TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }}
       TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}

.trivyignore.yaml

@@ -1,6 +1,6 @@
 misconfigurations:
-  # The tools Dockerfile doesn't need a healthcheck.
+  # Dockerfiles for ephemeral containers don't need a healthcheck.
   - id: AVD-DS-0026
     paths:
-      - Dockerfile.exporter # ephemeral container, healthcheck not necessary
+      - Dockerfile.exporter
       - Dockerfile.tools

Dockerfile.exporter

@@ -19,4 +19,8 @@ ENV PYTHONPATH=/opt/senzing/er/sdk/python:/app
 # Flush buffer - helps with print statements.
 ENV PYTHONUNBUFFERED=1
 
+# Define volumes necessary to support a read-only root filesystem on ECS
+# Fargate.
+VOLUME ["/home/senzing", "/var/lib/amazon", "/var/log"]
+
 CMD ["python3", "exporter.py"]

tofu/config/service/main.tf

@@ -35,6 +35,7 @@ module "system" {
   deletion_protection                = var.deletion_protection
   image_tag                          = local.image_tag
   image_tags_mutable                 = var.image_tags_mutable
+  log_level                          = var.log_level
 
   consumer_container_count = var.consumer_container_count
   consumer_cpu             = var.consumer_cpu

tofu/config/service/variables.tf

@@ -80,6 +80,17 @@ variable "key_recovery_period" {
   }
 }
 
+variable "log_level" {
+  type        = string
+  description = "Log level for all containers."
+  default     = "INFO"
+
+  validation {
+    condition     = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.log_level)
+    error_message = "Valid log levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL."
+  }
+}
+
 variable "program" {
   type        = string
   description = "Program the application belongs to."

tofu/modules/ephemeral_service/docker.tf

@@ -11,8 +11,8 @@ resource "docker_image" "container" {
     ]
 
     auth_config {
-      host_name  = data.aws_ecr_authorization_token.token.proxy_endpoint
-      password = data.aws_ecr_authorization_token.token.password
+      host_name = data.aws_ecr_authorization_token.token.proxy_endpoint
+      password  = data.aws_ecr_authorization_token.token.password
       user_name = data.aws_ecr_authorization_token.token.user_name
     }
   }