diff --git a/.github/actions/setup-opentofu/action.yaml b/.github/actions/setup-opentofu/action.yaml index b3bbdef..ce064a0 100644 --- a/.github/actions/setup-opentofu/action.yaml +++ b/.github/actions/setup-opentofu/action.yaml @@ -24,13 +24,19 @@ runs: run: tofu version - name: Set optional variables shell: bash + env: + TF_VAR_REGION: ${{ env.AWS_REGION }} + # For any of the defined variables that have a value set into TF_VAR_* + # (all uppercase), we set the corresponding TF_VAR_* (lowercase) variable + # that OpenTofu expects. run: | variables=( "apply_database_updates_immediately" "consumer_container_count" "consumer_cpu" "consumer_memory" "database_instance_count" "database_skip_final_snapshot" "deletion_protection" "deployment_environments" "environment" "export_expiration" - "image_tags_mutable" "key_recovery_period" "program" "project" "repository" + "image_tags_mutable" "key_recovery_period" "log_level" "program" + "project" "region" "repository" ) for var in ${variables[@]}; do name="TF_VAR_$(echo $var | tr '[:lower:]' '[:upper:]')" diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index ec73d24..914c3b7 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -51,6 +51,7 @@ jobs: TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }} TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} @@ -65,6 +66,7 @@ jobs: needs: plan environment: ${{ inputs.environment || 'development' }} env: + AWS_REGION: ${{ secrets.AWS_REGION }} TF_VAR_image_tag: ${{ inputs.image_tag || github.sha }} # Set required variables. TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} @@ -100,6 +102,7 @@ jobs: TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }} TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} diff --git a/.github/workflows/export.yaml b/.github/workflows/export.yaml new file mode 100644 index 0000000..9911fe3 --- /dev/null +++ b/.github/workflows/export.yaml @@ -0,0 +1,92 @@ +name: Trigger an export from Senzing to S3 + +on: + workflow_dispatch: + inputs: + environment: + description: Environment to run the exporter in. + default: development + required: true + type: environment + +permissions: + contents: read + id-token: write + +jobs: + launch: + name: Trigger export in ${{ inputs.environment }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + env: + # Set required variables. + TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} + TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }} + TF_VAR_vpc_private_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }} + TF_VAR_vpc_public_subnet_cidrs: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION || 'us-west-1' }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: GitHub_to_AWS_via_FederatedOIDC + - name: Setup OpenTofu + uses: ./.github/actions/setup-opentofu + env: + TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: ${{ secrets.TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY }} + TF_VAR_CONSUMER_CONTAINER_COUNT: ${{ secrets.TF_VAR_CONSUMER_CONTAINER_COUNT }} + TF_VAR_CONSUMER_CPU: ${{ secrets.TF_VAR_CONSUMER_CPU }} + TF_VAR_CONSUMER_MEMORY: ${{ secrets.TF_VAR_CONSUMER_MEMORY }} + TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT: ${{ secrets.TF_VAR_DATABASE_SKIP_FINAL_SNAPSHOT }} + TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }} + TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }} + TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }} + TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }} + TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} + TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} + TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} + TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }} + TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} + TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} + TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }} + TF_VAR_VPC_CIDR: ${{ secrets.TF_VAR_VPC_CIDR }} + TF_VAR_VPC_PRIVATE_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PRIVATE_SUBNET_CIDRS }} + TF_VAR_VPC_PUBLIC_SUBNET_CIDRS: ${{ secrets.TF_VAR_VPC_PUBLIC_SUBNET_CIDRS }} + with: + config: service + - name: Get OpenTofu outputs + id: outputs + working-directory: ./tofu/config/service + run: | + OUTPUTS=$(tofu output -json | jq -c) + echo "OUTPUTS=$OUTPUTS" + echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT + - name: Parse subnets + id: subnets + env: + SUBNETS: ${{ toJson(fromJson(steps.outputs.outputs.outputs).container_subnets.value) }} + run: | + SUBNET_STRING=$(echo "$SUBNETS" | jq -r '.[]') + echo "subnets<> $GITHUB_OUTPUT + echo "$SUBNET_STRING" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Launch container + id: run-task + uses: geekcell/github-action-aws-ecs-run-task@v5 + with: + cluster: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }} + task-definition: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter + override-container: ${{ secrets.TF_VAR_PROJECT }}-${{ secrets.TF_VAR_ENVIRONMENT }}-exporter + assign-public-ip: DISABLED + tail-logs: true + task-wait-until-stopped: true + # The block style indicator (|) is necessary to tell YAML to preserve + # newlines. + subnet-ids: | + ${{ steps.subnets.outputs.subnets }} + security-group-ids: | + ${{ fromJson(steps.outputs.outputs.outputs).task_security_group_id.value }} diff --git a/.github/workflows/launch-tools.yaml b/.github/workflows/launch-tools.yaml index cebaff9..6b8c59f 100644 --- a/.github/workflows/launch-tools.yaml +++ b/.github/workflows/launch-tools.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: environment: - description: Environment to destroy. + description: Environment to launch into. default: development required: true type: environment @@ -26,6 +26,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.environment }} env: + AWS_REGION: ${{ secrets.AWS_REGION }} # Set required variables. TF_VAR_repo_oidc_arn: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} TF_VAR_vpc_cidr: ${{ secrets.TF_VAR_VPC_CIDR }} @@ -51,10 +52,11 @@ jobs: TF_VAR_DATABASE_INSTANCE_COUNT: ${{ secrets.TF_VAR_DATABASE_INSTANCE_COUNT }} TF_VAR_DELETION_PROTECTION: ${{ secrets.TF_VAR_DELETION_PROTECTION }} TF_VAR_DEPLOYMENT_ENVIRONMENTS: ${{ secrets.TF_VAR_DEPLOYMENT_ENVIRONMENTS }} - TF_VAR_ENVIRONMENT: ${{ inputs.environment }} + TF_VAR_ENVIRONMENT: ${{ secrets.TF_VAR_ENVIRONMENT }} TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }} TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} diff --git a/.github/workflows/plan.yaml b/.github/workflows/plan.yaml index b09c244..70451e7 100644 --- a/.github/workflows/plan.yaml +++ b/.github/workflows/plan.yaml @@ -18,8 +18,14 @@ on: required: false type: string secrets: + # Required secrets. AWS_REGION: AWS_ROLE_ARN: + TF_VAR_REPO_OIDC_ARN: + TF_VAR_VPC_CIDR: + TF_VAR_VPC_PRIVATE_SUBNET_CIDRS: + TF_VAR_VPC_PUBLIC_SUBNET_CIDRS: + # Optional secrets. TF_VAR_APPLY_DATABASE_UPDATES_IMMEDIATELY: required: false TF_VAR_CONSUMER_CONTAINER_COUNT: @@ -42,16 +48,14 @@ on: required: false TF_VAR_KEY_RECOVERY_PERIOD: required: false + TF_VAR_LOG_LEVEL: + required: false TF_VAR_PROGRAM: required: false TF_VAR_PROJECT: required: false - TF_VAR_REPO_OIDC_ARN: TF_VAR_REPOSITORY: required: false - TF_VAR_VPC_CIDR: - TF_VAR_VPC_PRIVATE_SUBNET_CIDRS: - TF_VAR_VPC_PUBLIC_SUBNET_CIDRS: workflow_dispatch: inputs: config: @@ -113,6 +117,7 @@ jobs: TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} TF_VAR_PROJECT: ${{ secrets.TF_VAR_PROJECT }} TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 77d3728..2c9af5f 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -75,6 +75,7 @@ jobs: TF_VAR_EXPORT_EXPIRATION: ${{ secrets.TF_VAR_EXPORT_EXPIRATION }} TF_VAR_IMAGE_TAGS_MUTABLE: ${{ secrets.TF_VAR_IMAGE_TAGS_MUTABLE }} TF_VAR_KEY_RECOVERY_PERIOD: ${{ secrets.TF_VAR_KEY_RECOVERY_PERIOD }} + TF_VAR_LOG_LEVEL: ${{ secrets.TF_VAR_LOG_LEVEL }} TF_VAR_PROGRAM: ${{ secrets.TF_VAR_PROGRAM }} TF_VAR_REPO_OIDC_ARN: ${{ secrets.TF_VAR_REPO_OIDC_ARN }} TF_VAR_REPOSITORY: ${{ secrets.TF_VAR_REPOSITORY }} diff --git a/.trivyignore.yaml b/.trivyignore.yaml index ab26423..a26eb39 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -1,6 +1,6 @@ misconfigurations: - # The tools Dockerfile doesn't need a healthcheck. + # Dockerfiles for ephemeral containers don't need a healthcheck. - id: AVD-DS-0026 paths: - - Dockerfile.exporter # ephemeral container, healthcheck not necessary + - Dockerfile.exporter - Dockerfile.tools diff --git a/Dockerfile.exporter b/Dockerfile.exporter index 8e2de83..42bdad7 100644 --- a/Dockerfile.exporter +++ b/Dockerfile.exporter @@ -19,4 +19,8 @@ ENV PYTHONPATH=/opt/senzing/er/sdk/python:/app # Flush buffer - helps with print statements. ENV PYTHONUNBUFFERED=1 +# Define volumes necessary to support a read-only root filesystem on ECS +# Fargate. +VOLUME ["/home/senzing", "/var/lib/amazon", "/var/log"] + CMD ["python3", "exporter.py"] diff --git a/tofu/config/service/main.tf b/tofu/config/service/main.tf index 0bb4c13..bf4093d 100644 --- a/tofu/config/service/main.tf +++ b/tofu/config/service/main.tf @@ -35,6 +35,7 @@ module "system" { deletion_protection = var.deletion_protection image_tag = local.image_tag image_tags_mutable = var.image_tags_mutable + log_level = var.log_level consumer_container_count = var.consumer_container_count consumer_cpu = var.consumer_cpu diff --git a/tofu/config/service/variables.tf b/tofu/config/service/variables.tf index c16f19b..a92825c 100644 --- a/tofu/config/service/variables.tf +++ b/tofu/config/service/variables.tf @@ -80,6 +80,17 @@ variable "key_recovery_period" { } } +variable "log_level" { + type = string + description = "Log level for all containers." + default = "INFO" + + validation { + condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.log_level) + error_message = "Valid log levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL." + } +} + variable "program" { type = string description = "Program the application belongs to." diff --git a/tofu/modules/ephemeral_service/docker.tf b/tofu/modules/ephemeral_service/docker.tf index c6f6616..320b2c6 100644 --- a/tofu/modules/ephemeral_service/docker.tf +++ b/tofu/modules/ephemeral_service/docker.tf @@ -11,8 +11,8 @@ resource "docker_image" "container" { ] auth_config { - host_name = data.aws_ecr_authorization_token.token.proxy_endpoint - password = data.aws_ecr_authorization_token.token.password + host_name = data.aws_ecr_authorization_token.token.proxy_endpoint + password = data.aws_ecr_authorization_token.token.password user_name = data.aws_ecr_authorization_token.token.user_name } } diff --git a/tofu/modules/system/ecs.tf b/tofu/modules/system/ecs.tf index 6733fda..7046865 100644 --- a/tofu/modules/system/ecs.tf +++ b/tofu/modules/system/ecs.tf @@ -85,13 +85,13 @@ module "ecs" { tags = var.tags } -module "tools" { - source = "../ephemeral_service" +module "consumer" { + source = "../persistent_service" depends_on = [aws_iam_policy.queue, aws_iam_policy.secrets] project = var.project environment = var.environment - service = "tools" + service = "consumer" image_tag = var.image_tag image_tags_mutable = var.image_tags_mutable force_delete = !var.deletion_protection @@ -100,38 +100,65 @@ module "tools" { otel_ssm_parameter_arn = module.otel_config.ssm_parameter_arn execution_policies = [aws_iam_policy.secrets.arn] task_policies = [aws_iam_policy.queue.arn] - dockerfile = "Dockerfile.tools" + security_groups = [module.task_security_group.security_group_id] + cluster_arn = module.ecs.arn + container_subnets = var.container_subnets + desired_containers = var.consumer_container_count + cpu = var.consumer_cpu + memory = var.consumer_memory + dockerfile = "Dockerfile.consumer" docker_context = "${path.module}/../../../" - ephemeral_volumes = { - senzing-home = "/home/senzing" - # We need these to support ecs exec with a read-only root filesystem. - aws-lib = "/var/lib/amazon" - logs = "/var/log" + + environment_variables = { + LOG_LEVEL : var.log_level + Q_URL : module.sqs.queue_url } + environment_secrets = { + SENZING_ENGINE_CONFIGURATION_JSON = module.senzing_config.ssm_parameter_arn + } + + tags = var.tags +} + +module "exporter" { + source = "../ephemeral_service" + depends_on = [aws_iam_policy.exports, aws_iam_policy.secrets] + + project = var.project + environment = var.environment + service = "exporter" + image_tag = var.image_tag + image_tags_mutable = var.image_tags_mutable + force_delete = !var.deletion_protection + container_key_arn = aws_kms_key.container.arn + logging_key_id = var.logging_key_arn + otel_ssm_parameter_arn = module.otel_config.ssm_parameter_arn + execution_policies = [aws_iam_policy.secrets.arn] + task_policies = [aws_iam_policy.exports.arn] + dockerfile = "Dockerfile.exporter" + docker_context = "${path.module}/../../../" + environment_variables = { - PGHOST : module.database.cluster_endpoint - PGSSLMODE : "require" Q_URL : module.sqs.queue_url - SENZING_DATASOURCES : "PEOPLE CUSTOMERS" + S3_BUCKET_NAME : module.s3.bucket + LOG_LEVEL : var.log_level } environment_secrets = { - PGPASSWORD : "${module.database.cluster_master_user_secret[0].secret_arn}:password::" - PGUSER : "${module.database.cluster_master_user_secret[0].secret_arn}:username::" SENZING_ENGINE_CONFIGURATION_JSON = module.senzing_config.ssm_parameter_arn } tags = var.tags } -module "consumer" { - source = "../persistent_service" +module "tools" { + source = "../ephemeral_service" depends_on = [aws_iam_policy.queue, aws_iam_policy.secrets] project = var.project environment = var.environment - service = "consumer" + service = "tools" image_tag = var.image_tag image_tags_mutable = var.image_tags_mutable force_delete = !var.deletion_protection @@ -140,20 +167,26 @@ module "consumer" { otel_ssm_parameter_arn = module.otel_config.ssm_parameter_arn execution_policies = [aws_iam_policy.secrets.arn] task_policies = [aws_iam_policy.queue.arn] - security_groups = [module.task_security_group.security_group_id] - cluster_arn = module.ecs.arn - container_subnets = var.container_subnets - desired_containers = var.consumer_container_count - cpu = var.consumer_cpu - memory = var.consumer_memory - dockerfile = "Dockerfile.consumer" + dockerfile = "Dockerfile.tools" docker_context = "${path.module}/../../../" + ephemeral_volumes = { + senzing-home = "/home/senzing" + # We need these to support ecs exec with a read-only root filesystem. + aws-lib = "/var/lib/amazon" + logs = "/var/log" + } environment_variables = { + LOG_LEVEL : var.log_level + PGHOST : module.database.cluster_endpoint + PGSSLMODE : "require" Q_URL : module.sqs.queue_url + SENZING_DATASOURCES : "PEOPLE CUSTOMERS" } environment_secrets = { + PGPASSWORD : "${module.database.cluster_master_user_secret[0].secret_arn}:password::" + PGUSER : "${module.database.cluster_master_user_secret[0].secret_arn}:username::" SENZING_ENGINE_CONFIGURATION_JSON = module.senzing_config.ssm_parameter_arn } diff --git a/tofu/modules/system/iam.tf b/tofu/modules/system/iam.tf index c9ebafa..f79ca2e 100644 --- a/tofu/modules/system/iam.tf +++ b/tofu/modules/system/iam.tf @@ -1,3 +1,19 @@ +resource "aws_iam_policy" "exports" { + name_prefix = "${local.prefix}-exports-access-" + description = "Allow access to the S3 bucket for Senzing exports." + + policy = jsonencode(yamldecode(templatefile("${path.module}/templates/exports-access-policy.yaml.tftpl", { + bucket_arn = module.s3.arn + kms_arn = aws_kms_key.queue.arn + }))) + + tags = var.tags + + lifecycle { + create_before_destroy = true + } +} + resource "aws_iam_policy" "queue" { name_prefix = "${local.prefix}-queue-access-" description = "Allow access to the SQS queues for Senzing." diff --git a/tofu/modules/system/templates/exports-access-policy.yaml.tftpl b/tofu/modules/system/templates/exports-access-policy.yaml.tftpl new file mode 100644 index 0000000..0cfffc1 --- /dev/null +++ b/tofu/modules/system/templates/exports-access-policy.yaml.tftpl @@ -0,0 +1,15 @@ +Version: '2012-10-17' +Statement: + - Sid: KeyAccess + Effect: Allow + Action: + - kms:Decrypt + - kms:GenerateDataKey + Resource: + - "${kms_arn}" + - Sid: S3Access + Effect: Allow + Action: + - s3:PutObject + Resource: + - "${bucket_arn}/*" diff --git a/tofu/modules/system/variables.tf b/tofu/modules/system/variables.tf index 223654b..3bf8e0a 100644 --- a/tofu/modules/system/variables.tf +++ b/tofu/modules/system/variables.tf @@ -129,6 +129,17 @@ variable "logging_bucket" { description = "S3 bucket to use for log collection." } +variable "log_level" { + type = string + description = "Log level for all containers." + default = "INFO" + + validation { + condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.log_level) + error_message = "Valid log levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL." + } +} + variable "logging_key_arn" { type = string description = "ARN of the KMS key to use for log encryption."