feat: Added consumer container. (#26)

jamesiarmes · web-flow · commit e89d030a4abc · 2025-09-26T12:56:13.000-04:00
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -41,6 +41,9 @@ jobs:
       AWS_REGION: ${{ secrets.AWS_REGION }}
       AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
       TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
+      TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
+      TF_VAR_CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
+      TF_VAR_CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
       TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
       TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
       TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
@@ -88,6 +91,9 @@ jobs:
           # For any of these that have a value, the corresponding TF_VAR_*
           # environment variable will be set.
           APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
+          TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
+          CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
+          CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
           DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
           DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
           DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
@@ -100,7 +106,8 @@ jobs:
           REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}
         run: |
           variables=(
-            "apply_database_updates_immediately" "database_skip_final_snapshot"
+            "apply_database_updates_immediately" "consumer_container_count"
+            "consumer_cpu" "consumer_memory" "database_skip_final_snapshot"
             "deletion_protection" "deployment_environments" "environment"
             "export_expiration" "image_tags_mutable" "key_recovery_period"
             "program" "project" "repository"
diff --git a/.github/workflows/plan.yaml b/.github/workflows/plan.yaml
@@ -22,6 +22,12 @@ on:
       AWS_ROLE_ARN:
       TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY:
         required: false
+      TF_VAR_CONSUMER_CONTAINER_COUNT:
+        required: false
+      TF_VAR_CONSUMER_CPU:
+        required: false
+      TF_VAR_CONSUMER_MEMORY:
+        required: false
       TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT:
         required: false
       TF_VAR_DELETION_PROTECTION:
@@ -99,6 +105,9 @@ jobs:
           # For any of these that have a value, the corresponding TF_VAR_*
           # environment variable will be set.
           APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
+          TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
+          CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
+          CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
           DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
           DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
           DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
@@ -111,7 +120,8 @@ jobs:
           REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }}
         run: |
           variables=(
-            "apply_database_updates_immediately" "database_skip_final_snapshot"
+            "apply_database_updates_immediately" "consumer_container_count"
+            "consumer_cpu" "consumer_memory" "database_skip_final_snapshot"
             "deletion_protection" "deployment_environments" "environment"
             "export_expiration" "image_tags_mutable" "key_recovery_period"
             "program" "project" "repository"
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -67,6 +67,9 @@ jobs:
       AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
       TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }}
       TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }}
+      TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }}
+      TF_VAR_CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }}
+      TF_VAR_CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }}
       TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }}
       TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }}
       TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }}
diff --git a/Dockerfile.consumer b/Dockerfile.consumer
@@ -19,4 +19,8 @@ ENV PYTHONPATH=/opt/senzing/er/sdk/python:/app
 # Flush buffer - helps with print statements.
 ENV PYTHONUNBUFFERED=1
 
+# Define volumes necessary to support a read-only root filesystem on ECS
+# Fargate.
+VOLUME ["/home/senzing", "/var/lib/amazon", "/var/log"]
+
 CMD ["python3", "consumer.py"]
diff --git a/tofu/config/service/main.tf b/tofu/config/service/main.tf
@@ -34,4 +34,8 @@ module "system" {
   deletion_protection                = var.deletion_protection
   image_tag                          = var.image_tag != null ? var.image_tag : sha256(timestamp())
   image_tags_mutable                 = var.image_tags_mutable
+
+  consumer_container_count = var.consumer_container_count
+  consumer_cpu             = var.consumer_cpu
+  consumer_memory          = var.consumer_memory
 }
diff --git a/tofu/config/service/variables.tf b/tofu/config/service/variables.tf
@@ -4,6 +4,24 @@ variable "apply_database_updates_immediately" {
   default     = false
 }
 
+variable "consumer_container_count" {
+  type        = number
+  description = "Desired number of consumer containers to run."
+  default     = 1
+}
+
+variable "consumer_cpu" {
+  type        = number
+  description = "Number of virtual CPUs to allocate to each consumer container."
+  default     = 1
+}
+
+variable "consumer_memory" {
+  type        = number
+  description = "Amount of memory (in MiB) to allocate to each consumer container."
+  default     = 4096
+}
+
 variable "database_skip_final_snapshot" {
   type        = bool
   description = "Whether to skip the final snapshot when the database cluster is deleted."
diff --git a/tofu/modules/ephemeral_service/main.tf b/tofu/modules/ephemeral_service/main.tf
@@ -24,7 +24,7 @@ module "ecs_task" {
   version = "~> 5.3"
 
   name   = local.prefix
-  cpu    = var.cpu
+  cpu    = (var.cpu * 1024)
   memory = var.memory
   # TODO: These roles?
   daemon_role              = aws_iam_role.execution.arn
@@ -37,8 +37,8 @@ module "ecs_task" {
   container_definitions = jsonencode(yamldecode(templatefile(
     "${path.module}/templates/container-definitions.yaml.tftpl", {
       name              = local.prefix
-      cpu               = var.cpu - 256
-      memory            = var.memory - 512
+      cpu               = (var.cpu * 1024)
+      memory            = var.memory
       image             = "${module.ecr.repository_url}:${var.image_tag}"
       container_command = var.container_command
       container_port    = var.container_port
diff --git a/tofu/modules/ephemeral_service/variables.tf b/tofu/modules/ephemeral_service/variables.tf
@@ -17,8 +17,8 @@ variable "container_port" {
 
 variable "cpu" {
   type        = number
-  description = "CPU unit for this task."
-  default     = 512
+  description = "Number of virtual CPUs to allocate to the container."
+  default     = 1
 }
 
 variable "docker_context" {
@@ -55,6 +55,7 @@ variable "ephemeral_volumes" {
   type        = map(string)
   description = "Map of ephemeral volume names to mount paths."
   default = {
+    logs         = "/var/log"
     senzing-home = "/home/senzing"
   }
 }
@@ -91,7 +92,7 @@ variable "logging_key_id" {
 variable "memory" {
   type        = number
   description = "Memory for this task."
-  default     = 1024
+  default     = 4096
 }
 
 variable "otel_ssm_parameter_arn" {
diff --git a/tofu/modules/persistent_service/main.tf b/tofu/modules/persistent_service/main.tf
@@ -0,0 +1,50 @@
+module "task" {
+  source = "../ephemeral_service"
+
+  project                  = var.project
+  service                  = var.service
+  image_tag                = var.image_tag
+  image_tags_mutable       = var.image_tags_mutable
+  force_delete             = var.force_delete
+  memory                   = var.memory
+  otel_ssm_parameter_arn   = var.otel_ssm_parameter_arn
+  otel_log_level           = var.otel_log_level
+  logging_key_id           = var.logging_key_id
+  task_policies            = var.task_policies
+  untagged_image_retention = var.untagged_image_retention
+  execution_policies       = var.execution_policies
+  container_key_arn        = var.container_key_arn
+  container_command        = var.container_command
+  docker_context           = var.docker_context
+  dockerfile               = var.dockerfile
+  environment_secrets      = var.environment_secrets
+  environment_variables    = var.environment_variables
+  ephemeral_volumes        = var.ephemeral_volumes
+  cpu                      = var.cpu
+
+  tags = var.tags
+}
+
+module "service" {
+  source  = "HENNGE/ecs/aws//modules/core/service"
+  version = "5.3.0"
+
+  cluster                = var.cluster_arn
+  name                   = join("-", compact([var.project, var.environment, var.service]))
+  create_task_definition = false
+  task_definition_arn    = module.task.task_definition_arn
+  desired_count          = var.desired_containers
+
+  launch_type                   = "FARGATE"
+  task_requires_compatibilities = ["FARGATE"]
+  enable_execute_command        = var.enable_execute_command
+  propagate_tags                = "SERVICE"
+
+  network_configuration = {
+    subnets          = var.container_subnets
+    security_groups  = var.security_groups
+    assign_public_ip = false
+  }
+
+  tags = var.tags
+}
diff --git a/tofu/modules/persistent_service/outputs.tf b/tofu/modules/persistent_service/outputs.tf
@@ -0,0 +1,19 @@
+output "container_name" {
+  description = "Name of the container in the ECS task definition."
+  value       = module.task.container_name
+}
+
+output "docker_push" {
+  description = "Commands to push a Docker image to the container repository."
+  value       = module.task.docker_push
+}
+
+output "service_name" {
+  description = "Name of the ECS service."
+  value       = module.service.name
+}
+
+output "task_definition_arn" {
+  description = "ARN of the ECS task definition."
+  value       = module.task.task_definition_arn
+}
diff --git a/tofu/modules/persistent_service/variables.tf b/tofu/modules/persistent_service/variables.tf
@@ -0,0 +1,157 @@
+variable "cluster_arn" {
+  type        = string
+  description = "ARN of the ECS cluster to deploy to."
+}
+
+variable "container_command" {
+  type        = list(string)
+  description = "Command to run in the container. Defaults to the image's CMD."
+  default     = []
+}
+
+variable "container_key_arn" {
+  type        = string
+  description = "ARN of the KMS key to use for encrypting the container image repository."
+}
+
+variable "container_subnets" {
+  type        = list(string)
+  description = "The IDs of the subnets in which the container resources should be deployed."
+}
+
+variable "cpu" {
+  type        = number
+  description = "Number of virtual CPUs to allocate to the container."
+  default     = 1
+}
+
+variable "desired_containers" {
+  type        = number
+  description = "Desired number of running containers for the service."
+  default     = 1
+}
+
+variable "docker_context" {
+  type        = string
+  description = "Path to the Docker build context."
+  default     = null
+}
+
+variable "dockerfile" {
+  type        = string
+  description = "Path to the Dockerfile to use for building the image."
+  default     = "Dockerfile"
+}
+
+variable "enable_execute_command" {
+  type        = bool
+  description = "Whether to enable ECS Exec on the service."
+  default     = false
+}
+
+variable "environment" {
+  type        = string
+  description = "Environment for the deployment."
+  default     = "dev"
+}
+
+variable "environment_secrets" {
+  type        = map(string)
+  description = "Secrets to be injected as environment variables on the container."
+  default     = {}
+}
+
+variable "environment_variables" {
+  type        = map(string)
+  description = "Environment variables to set on the container."
+  default     = {}
+}
+
+variable "ephemeral_volumes" {
+  type        = map(string)
+  description = "Map of ephemeral volume names to mount paths."
+  default = {
+    logs         = "/var/log"
+    senzing-home = "/home/senzing"
+  }
+}
+
+variable "execution_policies" {
+  type        = list(string)
+  description = "Additional policies to add to the task execution role."
+  default     = []
+}
+
+variable "force_delete" {
+  type        = bool
+  description = "Force deletion of resources. If changing to true, be sure to apply before destroying."
+  default     = false
+}
+
+variable "image_tag" {
+  type        = string
+  description = "Tag for the image to be deployed."
+  default     = "latest"
+}
+
+variable "image_tags_mutable" {
+  type        = bool
+  description = "Whether image tags in the repository can be mutated."
+  default     = false
+}
+
+variable "logging_key_id" {
+  type        = string
+  description = "KMS key ID for encrypting logs."
+}
+
+variable "memory" {
+  type        = number
+  description = "Memory for this task."
+  default     = 4096
+}
+
+variable "otel_ssm_parameter_arn" {
+  type        = string
+  description = "ARN of the SSM parameter containing the OpenTelemetry collector configuration."
+}
+
+variable "otel_log_level" {
+  type        = string
+  description = "Log level for the OpenTelemetry collector."
+  default     = "info"
+}
+
+variable "project" {
+  type        = string
+  description = "Project that these resources are supporting."
+}
+
+variable "security_groups" {
+  type        = list(string)
+  description = "The IDs of the security groups to associate with the container resources."
+  default     = []
+}
+
+variable "service" {
+  type        = string
+  description = "Service that these resources are supporting. Example: 'api', 'web', 'worker'"
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "Tags to apply to all resources."
+  default     = {}
+}
+
+variable "task_policies" {
+  type        = list(string)
+  description = "Additional policies to add to the task role."
+  default     = []
+}
+
+variable "untagged_image_retention" {
+  type        = number
+  description = "Retention period (after push) for untagged images, in days."
+  default     = 14
+}
diff --git a/tofu/modules/persistent_service/versions.tf b/tofu/modules/persistent_service/versions.tf
@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 1.6"
+}
diff --git a/tofu/modules/system/ecs.tf b/tofu/modules/system/ecs.tf
diff --git a/tofu/modules/system/iam.tf b/tofu/modules/system/iam.tf
diff --git a/tofu/modules/system/templates/queue-access-policy.yaml.tftpl b/tofu/modules/system/templates/queue-access-policy.yaml.tftpl
diff --git a/tofu/modules/system/variables.tf b/tofu/modules/system/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -34,4 +34,8 @@ module "system" {`
`34`	`34`	`deletion_protection = var.deletion_protection`
`35`	`35`	`image_tag = var.image_tag != null ? var.image_tag : sha256(timestamp())`
`36`	`36`	`image_tags_mutable = var.image_tags_mutable`
	`37`	`+`
	`38`	`+ consumer_container_count = var.consumer_container_count`
	`39`	`+ consumer_cpu = var.consumer_cpu`
	`40`	`+ consumer_memory = var.consumer_memory`
`37`	`41`	`}`
Original file line number	Diff line number	Diff line change
`@@ -17,8 +17,8 @@ variable "container_port" {`
`17`	`17`
`18`	`18`	`variable "cpu" {`
`19`	`19`	`type = number`
`20`		`- description = "CPU unit for this task."`
`21`		`- default = 512`
	`20`	`+ description = "Number of virtual CPUs to allocate to the container."`
	`21`	`+ default = 1`
`22`	`22`	`}`
`23`	`23`
`24`	`24`	`variable "docker_context" {`
`@@ -55,6 +55,7 @@ variable "ephemeral_volumes" {`
`55`	`55`	`type = map(string)`
`56`	`56`	`description = "Map of ephemeral volume names to mount paths."`
`57`	`57`	`default = {`
	`58`	`+ logs = "/var/log"`
`58`	`59`	`senzing-home = "/home/senzing"`
`59`	`60`	`}`
`60`	`61`	`}`
`@@ -91,7 +92,7 @@ variable "logging_key_id" {`
`91`	`92`	`variable "memory" {`
`92`	`93`	`type = number`
`93`	`94`	`description = "Memory for this task."`
`94`		`- default = 1024`
	`95`	`+ default = 4096`
`95`	`96`	`}`
`96`	`97`
`97`	`98`	`variable "otel_ssm_parameter_arn" {`