Skip to content

Commit 217fe41

Browse files
stephanmestephanme
authored
Merge pull request #368 from sap-contributions/cert_renewal_cronjob
Add K8s Cron Job to automatically renew certificates
2 parents 2d24774 + c157fd7 commit 217fe41

File tree

7 files changed

+199
-0
lines changed

7 files changed

+199
-0
lines changed

‎docs/concourse/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -252,3 +252,6 @@ Please see [DR scenario](disaster_recovery.md) for a fully automated recovery pr
252252

253253
## Automated secrets rotation for CloudSQL
254254
Please see [Secrets Rotation](secrets_rotation.md)
255+
256+
## Automated regeneration for certificates stored in CredHub
257+
Please see [Certificate Regeneration](certificate_regeneration.md)

‎docs/concourse/certificate_regeneration.md

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Automated certificate regeneration
2+
3+
You can deploy a K8s CronJob to automatically regenerate certificates which are stored in CredHub. A typical example are load balancer certificates used in a bosh-bootloader environment. The CronJob calls `credhub regenerate <certificate name>`. This will extend the certificate's validity while all other properties remain unchanged.
4+
5+
The automated regeneration is provided as separate Terragrunt module which must be deployed separately to enable the feature.
6+
7+
## Prerequisites
8+
9+
The certificate's CA must be stored in CredHub, and they must be correctly linked.
10+
11+
## Configuration and deployment
12+
13+
First, configure the list of certificates in your local `config.yaml`. Define one string with comma-separated certificate names, e.g.:
14+
```
15+
certificates_to_regenerate: "/concourse/main/cert_1,/concourse/main/cert_2"
16+
```
17+
18+
Next, change to the directory `terragrunt/<concourse-instance>/automatic_certificate_regeneration` and call
19+
```
20+
terragrunt apply --terragrunt-config cert_regen.hcl
21+
```
22+
You should see that Terraform creates a new resource:
23+
```
24+
resource "kubernetes_cron_job_v1" "automatic_certificate_regeneration"
25+
(...)
26+
```
27+
Confirm with `yes`. Afterward, you can see a new CronJob in your K8s deployment:
28+
```
29+
$ kubectl -n concourse get cronjobs
30+
NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE
31+
certificate-regeneration @monthly False 0 <none> 50m
32+
```
33+
To test the CronJob, you can invoke it explicitly and check the logs:
34+
```
35+
kubectl -n concourse create job --from=cronjob/certificate-regeneration cert-regen-job
36+
# wait a few seconds
37+
kubectl -n concourse get pods # search pod "cert-regen-job-<xyz>"
38+
kubectl -n concourse logs cert-regen-job-<xyz>
39+
```
40+
You should see the output from CredHub:
41+
```
42+
id: 68875a90-c1b7-4391-a2af-bd3a8f33ce47
43+
name: /concourse/main/cert_1
44+
type: certificate
45+
value: <redacted>
46+
version_created_at: "2024-05-07T12:23:43Z"
47+
(...)
48+
```
49+
50+
## Limitations
51+
52+
It's possible to renew CAs with the CronJob. Note however that this would be a one-step renewal process which can result in downtimes. The full 4-step CA renewal process as described on https://github.com/pivotal/credhub-release/blob/main/docs/ca-rotation.md is not implemented.
53+
54+
If you want to include the CA in the regeneration process, you can add it at the beginning of the list:
55+
```
56+
certificates_to_regenerate: "/concourse/main/my_CA,/concourse/main/cert_1,/concourse/main/cert_2"
57+
```
58+
The (self-signed) CA would be regenerated first and then the two certificates would be re-signed with the new CA and the validity would be extended.
59+
60+
## Deletion
61+
62+
To delete the CronJob, change to the directory `terragrunt/<concourse-instance>/automatic_certificate_regeneration` and call
63+
```
64+
terragrunt destroy --terragrunt-config cert_regen.hcl
65+
```

‎terraform-modules/concourse/automatic_certificate_regeneration/cron_job.tf

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
resource "kubernetes_cron_job_v1" "automatic_certificate_regeneration" {
2+
metadata {
3+
name = "certificate-regeneration"
4+
namespace = "concourse"
5+
}
6+
spec {
7+
schedule = "@monthly"
8+
failed_jobs_history_limit = 2
9+
successful_jobs_history_limit = 2
10+
job_template {
11+
metadata {}
12+
spec {
13+
template {
14+
metadata {}
15+
spec {
16+
restart_policy = "OnFailure"
17+
container {
18+
name = "cert-regen"
19+
image = "yatzek/credhub-cli:2.9.0"
20+
image_pull_policy = "IfNotPresent"
21+
command = ["bash", "-c", "IFS=',' read -r -a CERTIFICATES <<< \"$CERTS_TO_RENEW\"; for cert in \"$${CERTIFICATES[@]}\"; do credhub regenerate -n \"$cert\"; done"]
22+
env {
23+
name = "CERTS_TO_RENEW"
24+
value = var.certificates_to_regenerate
25+
}
26+
env {
27+
name = "CREDHUB_SERVER"
28+
value = "https://credhub.concourse.svc.cluster.local:9000"
29+
}
30+
env {
31+
name = "CREDHUB_CA_CERT"
32+
value_from {
33+
secret_key_ref {
34+
key = "certificate"
35+
name = "credhub-root-ca"
36+
}
37+
}
38+
}
39+
env {
40+
name = "CREDHUB_CLIENT"
41+
value = "credhub_admin_client"
42+
}
43+
env {
44+
name = "CREDHUB_SECRET"
45+
value_from {
46+
secret_key_ref {
47+
key = "password"
48+
name = "credhub-admin-client-credentials"
49+
}
50+
}
51+
}
52+
}
53+
}
54+
}
55+
}
56+
}
57+
}
58+
}

‎terraform-modules/concourse/automatic_certificate_regeneration/providers.tf

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
terraform {
2+
required_providers {
3+
kubernetes = {
4+
source = "hashicorp/kubernetes"
5+
}
6+
}
7+
}
8+
9+
provider "google" {
10+
project = var.project
11+
region = var.region
12+
zone = var.zone
13+
}
14+
15+
data "google_client_config" "provider" {}
16+
17+
data "google_container_cluster" "wg_ci" {
18+
project = var.project
19+
name = var.gke_name
20+
location = var.zone
21+
}
22+
23+
provider "kubernetes" {
24+
host = "https://${data.google_container_cluster.wg_ci.endpoint}"
25+
token = data.google_client_config.provider.access_token
26+
cluster_ca_certificate = base64decode(data.google_container_cluster.wg_ci.master_auth[0].cluster_ca_certificate)
27+
}

‎terraform-modules/concourse/automatic_certificate_regeneration/variables.tf

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
variable "project" { nullable = false }
2+
variable "region" { nullable = false }
3+
variable "zone" { nullable = false }
4+
5+
variable "gke_name" { nullable = false }
6+
7+
variable "certificates_to_regenerate" { nullable = false }

‎terragrunt/concourse-wg-ci-test/automatic_certificate_regeneration/cert_regen.hcl

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
locals {
2+
config = yamldecode(file("../config.yaml"))
3+
}
4+
5+
remote_state {
6+
backend = "gcs"
7+
generate = {
8+
path = "backend.tf"
9+
if_exists = "overwrite"
10+
}
11+
config = {
12+
bucket = "${local.config.gcs_bucket}"
13+
prefix = "${local.config.gcs_prefix}/automatic-certificate-regeneration"
14+
project = "${local.config.project}"
15+
location = "${local.config.region}"
16+
# use for uniform bucket-level access
17+
# (https://cloud.google.com/storage/docs/uniform-bucket-level-access)
18+
enable_bucket_policy_only = false
19+
}
20+
}
21+
22+
terraform {
23+
source = local.config.tf_modules.automatic_certificate_regeneration
24+
}
25+
26+
inputs = {
27+
project = local.config.project
28+
region = local.config.region
29+
zone = local.config.zone
30+
31+
gke_name = local.config.gke_name
32+
33+
certificates_to_regenerate = local.config.certificates_to_regenerate
34+
}

‎terragrunt/concourse-wg-ci-test/config.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ tf_modules:
4444
dr_restore: "../../..//terraform-modules/concourse/dr_restore"
4545
e2e_test: "../../..//terraform-modules/concourse/e2e_test"
4646
secret_rotation_postgresql: "../../..//terraform-modules/concourse/secret_rotation_postgresql"
47+
automatic_certificate_regeneration: "../../..//terraform-modules/concourse/automatic_certificate_regeneration"
4748

4849

4950
fly_team: main
@@ -122,3 +123,7 @@ wg_ci_cnrm_service_account_permissions: [
122123
"cloudsql.databases.list",
123124
"cloudsql.databases.update"
124125
]
126+
127+
# list of certificates that shall be automatically renewed every month
128+
# enter as one string with a comma-separated list of CredHub certificate names
129+
certificates_to_regenerate: "/concourse/main/test_cert"

0 commit comments

Comments
 (0)