feat: add log forwarder cf (#6)

* feat: add log forwarder cf
* fix: update version refs and readme

Author: Ic3w0lf

.github/.templatesyncignore

@@ -2,4 +2,8 @@ README.md
 .github/workflows/*
 .terraform-docs.yml
 docs/20-badges.md
+docs/assets/logo.svg
 *.tf
+test/*
+go.mod
+go.sum

README.md

@@ -50,6 +50,12 @@ for creating the AWS Integration role and the following submodules:
 ### Resource collection
 * Cloud Security Posture Management (can be enabled via the integration role)
 
+### ECS Fargate Agent
+* Scrape DB metrics for DBM
+
+### Log Forwarder Lambda
+* Forward any S3 or CloudWatch logs to Datadog
+
 ## Inputs
 
 | Name | Description | Type | Default | Required |
@@ -81,9 +87,9 @@ for creating the AWS Integration role and the following submodules:
 
 ## Resources
 
-- resource.aws_iam_role_policy_attachment.csp (main.tf#151)
-- resource.datadog_integration_aws.main (main.tf#22)
-- data source.aws_caller_identity.current (main.tf#14)
+- resource.aws_iam_role_policy_attachment.csp (main.tf#157)
+- resource.datadog_integration_aws.main (main.tf#28)
+- data source.aws_caller_identity.current (main.tf#20)
 
 # Examples
 ### Full
@@ -108,5 +114,14 @@ module "metric_stream" {
   prefix          = "datadog-pro"
   datadog_api_key = var.datadog_api_key
 }
+
+# Deploy the log forwarder Lambda via CloudFormation:
+# https://docs.datadoghq.com/logs/guide/forwarder/?tab=terraform
+module "log_forwarder" {
+  source = "../../modules/log_forwarder"
+
+  prefix          = "datadog-pro"
+  datadog_api_key = var.datadog_api_key
+}
 ```
 <!-- END_TF_DOCS -->

examples/complete/main.tf

@@ -18,3 +18,12 @@ module "metric_stream" {
   prefix          = "datadog-pro"
   datadog_api_key = var.datadog_api_key
 }
+
+# Deploy the log forwarder Lambda via CloudFormation:
+# https://docs.datadoghq.com/logs/guide/forwarder/?tab=terraform
+module "log_forwarder" {
+  source = "../../modules/log_forwarder"
+
+  prefix          = "datadog-pro"
+  datadog_api_key = var.datadog_api_key
+}

main.tf

@@ -10,6 +10,12 @@
  *
  * ### Resource collection
  * * Cloud Security Posture Management (can be enabled via the integration role)
+ *
+ * ### ECS Fargate Agent
+ * * Scrape DB metrics for DBM
+ *
+ * ### Log Forwarder Lambda
+ * * Forward any S3 or CloudWatch logs to Datadog
  */
 data "aws_caller_identity" "current" {
   count = var.aws_account_id == null ? 1 : 0

modules/fargate_agent/main.tf

@@ -6,7 +6,7 @@
  * having to setup an EC2 instance. For more information, see: https://docs.datadoghq.com/database_monitoring/
  */
 module "ecs_agent_container" {
-  source = "github.com/geekcell/terraform-aws-ecs-container-definition?ref=main"
+  source = "github.com/geekcell/terraform-aws-ecs-container-definition?ref=v1"
 
   name  = "datadog-agent"
   image = var.agent_container
@@ -26,7 +26,7 @@ module "ecs_agent_container" {
 }
 
 module "ecs_task_definition" {
-  source = "github.com/geekcell/terraform-aws-ecs-task-definition.git?ref=main"
+  source = "github.com/geekcell/terraform-aws-ecs-task-definition.git?ref=v1"
 
   name                   = var.name
   container_definitions  = [module.ecs_agent_container.hcl]
@@ -78,7 +78,7 @@ resource "aws_secretsmanager_secret" "main" {
 module "ecs_exec_ssm_policy" {
   count = length(var.secretsmanager_secret_keys) > 0 ? 1 : 0
 
-  source = "github.com/geekcell/terraform-aws-iam-policy?ref=main"
+  source = "github.com/geekcell/terraform-aws-iam-policy?ref=v1"
 
   name = "${var.name}-ssm-env"
   statements = [

modules/log_forwarder/.terraform-docs.yml

@@ -0,0 +1,19 @@
+content: |-
+  {{ .Header }}
+
+  {{ .Inputs }}
+
+  {{ .Outputs }}
+
+  {{ .Providers }}
+
+  ## Resources
+  {{ range .Module.Resources }}
+  - {{ .GetMode }}.{{ .Spec }} ({{ .Position.Filename }}#{{ .Position.Line }})
+  {{- end }}
+
+  # Examples
+    ### Full
+    ```hcl
+    {{ include "examples/full/main.tf" }}
+    ```

modules/log_forwarder/README.md

@@ -0,0 +1,60 @@
+<!-- BEGIN_TF_DOCS -->
+# Terraform AWS Datadog Log Forwarder Lambda Module
+
+This Terraform module deploys the Datadog Log Forwarder Lambda function as a Cloudformation stack. This is this
+recommended way by Datadog.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_datadog_api_key"></a> [datadog\_api\_key](#input\_datadog\_api\_key) | Pass an existing Datadog API key to write to the SecretsManager Secret. | `string` | `null` | no |
+| <a name="input_datadog_site"></a> [datadog\_site](#input\_datadog\_site) | Datadog site to use. | `string` | `"datadoghq.eu"` | no |
+| <a name="input_lambda_function_name"></a> [lambda\_function\_name](#input\_lambda\_function\_name) | The name of the Lambda function. | `string` | `null` | no |
+| <a name="input_lambda_log_retention_days"></a> [lambda\_log\_retention\_days](#input\_lambda\_log\_retention\_days) | The number of days to retain log events for the Lambda in Cloudwatch. | `number` | `90` | no |
+| <a name="input_lambda_memory_size"></a> [lambda\_memory\_size](#input\_lambda\_memory\_size) | Amount of memory to allocate to the Lambda function. | `number` | `1024` | no |
+| <a name="input_lambda_reserved_concurrency"></a> [lambda\_reserved\_concurrency](#input\_lambda\_reserved\_concurrency) | The number of concurrent executions reserved for the Lambda. | `number` | `null` | no |
+| <a name="input_lambda_timeout"></a> [lambda\_timeout](#input\_lambda\_timeout) | The amount of time the Lambda function has to run in seconds. | `number` | `120` | no |
+| <a name="input_prefix"></a> [prefix](#input\_prefix) | Prefix to use for resources in this module. | `string` | n/a | yes |
+| <a name="input_s3_bucket_name"></a> [s3\_bucket\_name](#input\_s3\_bucket\_name) | The name of the S3 forwarder bucket to create. | `string` | `null` | no |
+| <a name="input_s3_mfa"></a> [s3\_mfa](#input\_s3\_mfa) | MFA device ARN including a TOTP token to enable MFA delete for the forwarder bucket. | `string` | `null` | no |
+| <a name="input_s3_mfa_delete"></a> [s3\_mfa\_delete](#input\_s3\_mfa\_delete) | Enable MFA Delete for the forwarder bucket. | `string` | `"Disabled"` | no |
+| <a name="input_s3_noncurrent_version_expiration"></a> [s3\_noncurrent\_version\_expiration](#input\_s3\_noncurrent\_version\_expiration) | Specifies when non-current object versions expire in the forwarder bucket. | `number` | `30` | no |
+| <a name="input_s3_versioning"></a> [s3\_versioning](#input\_s3\_versioning) | Enable S3 Versioning for the forwarder bucket. | `string` | `"Enabled"` | no |
+| <a name="input_stack_additional_parameters"></a> [stack\_additional\_parameters](#input\_stack\_additional\_parameters) | Additional parameters to pass to the CloudFormation template. | `map(string)` | `{}` | no |
+| <a name="input_stack_capabilities"></a> [stack\_capabilities](#input\_stack\_capabilities) | CloudFormation capabilities required to create the stack. | `list(string)` | <pre>[<br>  "CAPABILITY_IAM",<br>  "CAPABILITY_NAMED_IAM",<br>  "CAPABILITY_AUTO_EXPAND"<br>]</pre> | no |
+| <a name="input_stack_template_url"></a> [stack\_template\_url](#input\_stack\_template\_url) | URL of the CloudFormation template to use to create the stack. | `string` | `"https://datadog-cloudformation-template.s3.amazonaws.com/aws/forwarder/latest.yaml"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | n/a | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_secretsmanager_secret_arn"></a> [secretsmanager\_secret\_arn](#output\_secretsmanager\_secret\_arn) | n/a |
+| <a name="output_stack_outputs"></a> [stack\_outputs](#output\_stack\_outputs) | n/a |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.50 |
+
+## Resources
+
+- resource.aws_cloudformation_stack.main (modules/log_forwarder/main.tf#21)
+- resource.aws_s3_bucket_lifecycle_configuration.main (modules/log_forwarder/main.tf#57)
+- resource.aws_s3_bucket_versioning.main (modules/log_forwarder/main.tf#45)
+- resource.aws_secretsmanager_secret.main (modules/log_forwarder/main.tf#7)
+- resource.aws_secretsmanager_secret_version.main (modules/log_forwarder/main.tf#14)
+
+# Examples
+### Full
+```hcl
+module "datadog_log_forwarder" {
+  source = "../../"
+
+  prefix          = "production-core"
+  datadog_api_key = "1234567890abcdef"
+}
+```
+<!-- END_TF_DOCS -->

modules/log_forwarder/examples/full/main.tf

@@ -0,0 +1,6 @@
+module "datadog_log_forwarder" {
+  source = "../../"
+
+  prefix          = "production-core"
+  datadog_api_key = "1234567890abcdef"
+}

modules/log_forwarder/examples/full/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.3"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.50"
+    }
+  }
+}

modules/log_forwarder/main.tf

@@ -0,0 +1,70 @@
+/**
+ * # Terraform AWS Datadog Log Forwarder Lambda Module
+ *
+ * This Terraform module deploys the Datadog Log Forwarder Lambda function as a Cloudformation stack. This is this
+ * recommended way by Datadog.
+ */
+resource "aws_secretsmanager_secret" "main" {
+  name        = "${var.prefix}-datadog-log-forwarder/api-key"
+  description = "Encrypted Datadog API Key."
+
+  tags = var.tags
+}
+
+resource "aws_secretsmanager_secret_version" "main" {
+  count = var.datadog_api_key != null ? 1 : 0
+
+  secret_id     = aws_secretsmanager_secret.main.id
+  secret_string = var.datadog_api_key
+}
+
+resource "aws_cloudformation_stack" "main" {
+  name         = "${var.prefix}-datadog-log-forwarder"
+  capabilities = var.stack_capabilities
+  template_url = var.stack_template_url
+
+  parameters = merge({
+    # Datadog
+    DdApiKeySecretArn = aws_secretsmanager_secret.main.arn
+    DdSite            = var.datadog_site
+
+    # Lambda
+    FunctionName        = coalesce(var.lambda_function_name, "${var.prefix}-datadog-log-forwarder")
+    MemorySize          = var.lambda_memory_size
+    Timeout             = var.lambda_timeout
+    LogRetentionInDays  = var.lambda_log_retention_days
+    ReservedConcurrency = var.lambda_reserved_concurrency
+
+    # Bucket
+    DdForwarderBucketName = coalesce(var.s3_bucket_name, "${var.prefix}-datadog-log-forwarder")
+  }, var.stack_additional_parameters)
+
+  tags = var.tags
+}
+
+resource "aws_s3_bucket_versioning" "main" {
+  bucket = coalesce(var.s3_bucket_name, "${var.prefix}-datadog-log-forwarder")
+  mfa    = var.s3_mfa
+
+  versioning_configuration {
+    status     = var.s3_versioning
+    mfa_delete = var.s3_mfa_delete
+  }
+
+  depends_on = [aws_cloudformation_stack.main]
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "main" {
+  bucket = coalesce(var.s3_bucket_name, "${var.prefix}-datadog-log-forwarder")
+
+  rule {
+    id     = "expire-non-current-versions"
+    status = "Enabled"
+
+    noncurrent_version_expiration {
+      noncurrent_days = var.s3_noncurrent_version_expiration
+    }
+  }
+
+  depends_on = [aws_cloudformation_stack.main]
+}

modules/log_forwarder/outputs.tf

@@ -0,0 +1,7 @@
+output "secretsmanager_secret_arn" {
+  value = aws_secretsmanager_secret.main.arn
+}
+
+output "stack_outputs" {
+  value = aws_cloudformation_stack.main.outputs
+}

modules/log_forwarder/variables.tf

@@ -0,0 +1,104 @@
+## NAMING
+variable "prefix" {
+  description = "Prefix to use for resources in this module."
+  type        = string
+}
+
+variable "tags" {
+  default = {}
+  type    = map(string)
+}
+
+## DATADOG
+variable "datadog_site" {
+  description = "Datadog site to use."
+  default     = "datadoghq.eu"
+  type        = string
+}
+
+variable "datadog_api_key" {
+  description = "Pass an existing Datadog API key to write to the SecretsManager Secret."
+  default     = null
+  type        = string
+}
+
+## STACK
+variable "stack_capabilities" {
+  description = "CloudFormation capabilities required to create the stack."
+  default     = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"]
+  type        = list(string)
+}
+
+variable "stack_template_url" {
+  description = "URL of the CloudFormation template to use to create the stack."
+  default     = "https://datadog-cloudformation-template.s3.amazonaws.com/aws/forwarder/latest.yaml"
+  type        = string
+}
+
+variable "stack_additional_parameters" {
+  description = "Additional parameters to pass to the CloudFormation template."
+  default     = {}
+  type        = map(string)
+}
+
+## LAMBDA
+variable "lambda_function_name" {
+  description = "The name of the Lambda function."
+  default     = null
+  type        = string
+}
+
+variable "lambda_reserved_concurrency" {
+  description = "The number of concurrent executions reserved for the Lambda."
+  default     = null
+  type        = number
+}
+
+variable "lambda_log_retention_days" {
+  description = "The number of days to retain log events for the Lambda in Cloudwatch."
+  default     = 90
+  type        = number
+}
+
+variable "lambda_timeout" {
+  description = "The amount of time the Lambda function has to run in seconds."
+  default     = 120
+  type        = number
+}
+
+variable "lambda_memory_size" {
+  description = "Amount of memory to allocate to the Lambda function."
+  default     = 1024
+  type        = number
+}
+
+## S3
+variable "s3_bucket_name" {
+  description = "The name of the S3 forwarder bucket to create."
+  default     = null
+  type        = string
+}
+
+variable "s3_versioning" {
+  description = "Enable S3 Versioning for the forwarder bucket."
+  default     = "Enabled"
+  type        = string
+}
+
+variable "s3_noncurrent_version_expiration" {
+  description = "Specifies when non-current object versions expire in the forwarder bucket."
+  default     = 30
+  type        = number
+}
+
+variable "s3_mfa" {
+  description = "MFA device ARN including a TOTP token to enable MFA delete for the forwarder bucket."
+  default     = null
+  type        = string
+}
+
+variable "s3_mfa_delete" {
+  description = "Enable MFA Delete for the forwarder bucket."
+  default     = "Disabled"
+  type        = string
+}

modules/log_forwarder/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.3"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.50"
+    }
+  }
+}