feat: Refactor infrastrucure to use layered configs rather than a monoconfig. (#10)

- Added foundation layer for infrastructure.
- Added networking layer for infrastructure.
- Added service layer for infrastructure.
- Use SSM to pass parameters between layers.
- Removed "development" monoconfig.
- Updated plan, deploy, and pull request workflows to supported layered approach.

Author: jamesiarmes

.github/workflows/deploy.yaml

@@ -3,6 +3,15 @@ name: Deploy infrastructure changes
 on:
   workflow_dispatch:
     inputs:
+      config:
+        description: OpenTofu configuration to deploy.
+        default: service
+        required: true
+        type: choice
+        options:
+          - foundation
+          - networking
+          - service
       environment:
         description: Environment to deploy to.
         default: development
@@ -13,9 +22,6 @@ permissions:
   contents: read
   id-token: write
 
-env:
-  CONFIG_PATH: ./tofu/config/development
-
 # TODO: Add an approval step between plan and deploy.
 jobs:
   plan:
@@ -79,11 +85,11 @@ jobs:
       - name: Download plan file
         uses: actions/download-artifact@v4
         with:
-          name: tfplan
-          path: ${{ env.CONFIG_PATH }}
+          name: ${{ inputs.config }}-tfplan
+          path: ./tofu/config/${{ inputs.config }}
       - name: Initialize OpenTofu
-        working-directory: ${{ env.CONFIG_PATH }}
+        working-directory: ./tofu/config/${{ inputs.config }}
         run: tofu init
       - name: Deploy changes
-        working-directory: ${{ env.CONFIG_PATH }}
+        working-directory: ./tofu/config/${{ inputs.config }}
         run: tofu apply tfplan

.github/workflows/plan.yaml

@@ -3,6 +3,11 @@ name: Plan infrastructure changes
 on:
   workflow_call:
     inputs:
+      config:
+        description: OpenTofu configuration to plan.
+        default: service
+        required: true
+        type: string
       environment:
         description: Environment to plan on.
         default: development
@@ -22,6 +27,15 @@ on:
       TF_VAR_VPC_PUBLIC_SUBNET_CIDRS:
   workflow_dispatch:
     inputs:
+      config:
+        description: OpenTofu configuration to plan.
+        default: service
+        required: true
+        type: choice
+        options:
+          - foundation
+          - networking
+          - service
       environment:
         description: Environment to plan on.
         default: development
@@ -32,12 +46,9 @@ permissions:
   contents: read
   id-token: write
 
-env:
-  CONFIG_PATH: ./tofu/config/development
-
 jobs:
   plan:
-    name: Plan changes to ${{ inputs.environment || 'development' }}
+    name: Plan changes to ${{ inputs.config }} in ${{ inputs.environment || 'development' }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment || 'development' }}
     env:
@@ -77,19 +88,19 @@ jobs:
             fi
           done
       - name: Initialize OpenTofu
-        working-directory: ${{ env.CONFIG_PATH }}
+        working-directory: ./tofu/config/${{ inputs.config }}
         run: tofu init
       - name: Plan changes
-        working-directory: ${{ env.CONFIG_PATH }}
+        working-directory: ./tofu/config/${{ inputs.config }}
         run: tofu plan -concise -no-color -out tfplan > plan.txt
       - name: Display plan
-        working-directory: ${{ env.CONFIG_PATH }}
+        working-directory: ./tofu/config/${{ inputs.config }}
         run: tofu show -plan tfplan
       - name: Upload plan file
         uses: actions/upload-artifact@v4
         with:
-          name: tfplan
+          name: ${{ inputs.config }}-tfplan
           path: |
-            ${{ env.CONFIG_PATH }}/plan.txt
-            ${{ env.CONFIG_PATH }}/tfplan
+            ./tofu/config/${{ inputs.config }}/plan.txt
+            ./tofu/config/${{ inputs.config }}/tfplan
           retention-days: 5

.github/workflows/pull-request.yaml

@@ -4,13 +4,61 @@ on:
   pull_request:
 
 jobs:
+  configs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+      - name: Find all configs
+        id: find
+        uses: bendrucker/find-terraform-modules@v1
+        with:
+          working-directory: tofu/config
+      - name: Show all matching configs
+        shell: bash
+        run: |
+          mods=(${{ join(fromJSON(steps.find.outputs.modules), ' ') }})
+          printf "%s\n" "${mods[@]}"
+      - name: Find all changed files
+        id: diff
+        uses: technote-space/get-diff-action@v6
+        with:
+          FORMAT: json
+      - name: Show changed files
+        run: |
+          echo "${{ steps.diff.outputs.diff }}"
+      - name: Get the modified configs
+        id: modified
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const configs = ${{ steps.find.outputs.modules }}
+            const diff = ${{ steps.diff.outputs.diff }}
+            const modifiedConfigs = configs.filter(
+              (config) => {
+                return !!diff.find(file => new RegExp(`^${config}/.+`).test(file))
+              }
+            ).map(config => config.replace(/^tofu\/config\//, ''))
+
+            core.setOutput('configs', modifiedConfigs)
+      - name: Show modified configs
+        run: |
+          echo "${{ steps.modified.outputs.configs }}"
+    outputs:
+      configs: ${{ steps.modified.outputs.configs }}
+
   plan:
     uses: ./.github/workflows/plan.yaml
+    needs: configs
     permissions:
       contents: read
       id-token: write
+    strategy:
+      matrix:
+        config: ${{ fromJson(needs.configs.outputs.configs) }}
     with:
       environment: development
+      config: ${{ matrix.config }}
     secrets:
       AWS_REGION: ${{ secrets.AWS_REGION }}
       AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
@@ -23,17 +71,20 @@ jobs:
 
   comment:
     runs-on: ubuntu-latest
-    needs: plan
+    needs:
+      - configs
+      - plan
     permissions:
       contents: read
       pull-requests: write
-    env:
-      CONFIG_PATH: ./tofu/config/development
+    strategy:
+      matrix:
+        config: ${{ fromJson(needs.configs.outputs.configs) }}
     steps:
       - name: Download plan file
         uses: actions/download-artifact@v4
         with:
-          name: tfplan
+          name: ${{ matrix.config }}-tfplan
       - uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -45,15 +96,15 @@ jobs:
               issue_number: context.issue.number,
             })
             const botComment = comments.find(comment => {
-              return comment.user.type === 'Bot' && comment.body.includes('## Plan output')
+              return comment.user.type === 'Bot' && comment.body.includes('## Plan output for ${{ matrix.config }} config')
             })
 
             // Read the contents of the plan.
             const fs = require('fs');
             const plan = fs.readFileSync('./plan.txt', 'utf8');
 
             // Prepare the format of the comment.
-            const output = `## Plan output\n\n\`\`\`\n${plan}\n\`\`\``
+            const output = `## Plan output for ${{ matrix.config }} config\n\n\`\`\`\n${plan}\n\`\`\``
 
             // If we have a comment, update it. Otherwise, create a new one.
             if (botComment) {

tofu/config/development/.terraform.lock.hcl

@@ -1,7 +1,7 @@
 terraform {
   backend "s3" {
     bucket         = "${var.project}-${var.environment}-tfstate"
-    key            = "${var.project}.tfstate"
+    key            = "foundation.tfstate"
     region         = var.region
     dynamodb_table = "${var.environment}.tfstate"
   }
@@ -24,37 +24,21 @@ module "backend" {
 module "logging" {
   source = "github.com/codeforamerica/tofu-modules-aws-logging?ref=2.1.1"
 
-  project                  = var.project
-  environment              = var.environment
-  cloudwatch_log_retention = 1
-  key_recovery_period      = 7
+  project             = var.project
+  environment         = var.environment
+  key_recovery_period = var.key_recovery_period
 
   tags = resource.aws_servicecatalogappregistry_application.application.application_tag
 }
 
-# TODO: Air gap this VPC from the internet.
-module "vpc" {
-  source = "github.com/codeforamerica/tofu-modules-aws-vpc?ref=1.1.2"
-
-  project         = var.project
-  environment     = var.environment
-  cidr            = var.vpc_cidr
-  logging_key_id  = module.logging.kms_key_arn
-  private_subnets = var.vpc_private_subnet_cidrs
-
-  # TODO: We don't need public subnets or a NAT gateway for an air gapped VPC.
-  public_subnets     = var.vpc_public_subnet_cidrs
-  single_nat_gateway = true
-
-  tags = resource.aws_servicecatalogappregistry_application.application.application_tag
-}
+module "outputs" {
+  source = "../../modules/outputs"
 
-module "system" {
-  source = "../../modules/system"
+  prefix = "/${var.project}/${var.environment}"
 
-  environment         = var.environment
-  project             = var.project
-  export_expiration   = var.export_expiration
-  key_recovery_period = var.key_recovery_period
-  logging_bucket      = module.logging.bucket
+  outputs = {
+    "application/tag" = aws_servicecatalogappregistry_application.application.application_tag["awsApplication"]
+    "logging/bucket"  = module.logging.bucket
+    "logging/key"     = module.logging.kms_key_arn
+  }
 }

tofu/config/foundation/.terraform.lock.hcl

@@ -0,0 +1,19 @@
+output "application_arn" {
+  description = "ARN of the Service Catalog App Registry application."
+  value       = aws_servicecatalogappregistry_application.application.arn
+}
+
+output "logging_bucket" {
+  value       = module.logging.bucket
+  description = "The name of the S3 bucket for logging."
+}
+
+output "logging_key_arn" {
+  value       = module.logging.kms_key_arn
+  description = "The ARN of the KMS key for logging."
+}
+
+output "state_bucket" {
+  value       = module.backend.bucket
+  description = "The name of the S3 bucket for infrastructure state files."
+}

tofu/config/development/main.tf renamed to tofu/config/foundation/main.tf

@@ -0,0 +1,34 @@
+variable "environment" {
+  type        = string
+  description = "Environment for the deployment."
+  default     = "development"
+}
+
+variable "key_recovery_period" {
+  type        = number
+  default     = 30
+  description = "Recovery period for deleted KMS keys in days. Must be between 7 and 30."
+
+  validation {
+    condition     = var.key_recovery_period > 6 && var.key_recovery_period < 31
+    error_message = "Recovery period must be between 7 and 30."
+  }
+}
+
+variable "program" {
+  type        = string
+  description = "Program the project belongs to."
+  default     = null
+}
+
+variable "project" {
+  type        = string
+  description = "Project that these resources are supporting."
+  default     = "sqs-senzing"
+}
+
+variable "region" {
+  type        = string
+  description = "AWS region where resources should be deployed."
+  default     = "us-west-1"
+}