Configure liveness and readiness probes for controller deployments (#508)

Part of aws-controllers-k8s/community#2012

This patch enhances controller availability by introducing liveness and
readiness probes. By configuring /healthz and /readyz endpoints for all
the controllers, the deployments can now effectively monitor their
health status.

While it's still a simple ping/pong healthz endpoints. We're laying the
ground for future more complex healthz and readiness checks. Thinking
something like checking that the CARM informer health...

Signed-off-by: Amine Hilaly <[email protected]>

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: a-hilaly

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/aws-controllers-k8s/pkg v0.0.9
-	github.com/aws-controllers-k8s/runtime v0.30.0
+	github.com/aws-controllers-k8s/runtime v0.30.1-0.20240217174305-eb1315466efb
 	github.com/aws/aws-sdk-go v1.49.0
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	// pin to v0.1.1 due to release problem with v0.1.2
@@ -20,7 +20,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/mod v0.14.0
 	k8s.io/apimachinery v0.29.0
-	sigs.k8s.io/controller-runtime v0.17.0
+	sigs.k8s.io/controller-runtime v0.17.2
 )
 
 require (

go.sum

@@ -73,8 +73,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws-controllers-k8s/pkg v0.0.9 h1:f3ejpb6VKnt5CS8F/0+7dGH691eox0xT9mQ2PW4ZrR4=
 github.com/aws-controllers-k8s/pkg v0.0.9/go.mod h1:VvdjLWmR6IJ3KU8KByKiq/lJE8M+ur2piXysXKTGUS0=
-github.com/aws-controllers-k8s/runtime v0.30.0 h1:AibYRdi/7xUA3t8BA0u8g+J+OioaTAT6R4Vq8hxLiYw=
-github.com/aws-controllers-k8s/runtime v0.30.0/go.mod h1:Pv1ozlUaO11KO2mwPN/HzhAtZ70ZDE9UP24mjsbkul0=
+github.com/aws-controllers-k8s/runtime v0.30.1-0.20240217174305-eb1315466efb h1:Cdicq3reak7cXeuSYpJJEf8lyEHcE/ACnevC133vWGc=
+github.com/aws-controllers-k8s/runtime v0.30.1-0.20240217174305-eb1315466efb/go.mod h1:6qr9ULkjOHo0fTwEUkE+48IxHqNbHxvvf/9JzGoR8pM=
 github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY=
 github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -1037,8 +1037,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4=
-sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s=
-sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
+sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0=
+sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
 sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=

templates/cmd/controller/main.go.tpl

@@ -17,6 +17,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrlrt "sigs.k8s.io/controller-runtime"
 	ctrlrtcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlrthealthz "sigs.k8s.io/controller-runtime/pkg/healthz"
 	ctrlrtmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	ctrlrtwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -122,6 +123,9 @@ func main() {
 		LeaderElection:          ackCfg.EnableLeaderElection,
 		LeaderElectionID:        "ack-" + awsServiceAPIGroup,
 		LeaderElectionNamespace: ackCfg.LeaderElectionNamespace,
+		HealthProbeBindAddress:  ackCfg.HealthzAddr,
+		LivenessEndpointName:    "/healthz",
+		ReadinessEndpointName:   "/readyz",
 	})
 	if err != nil {
 		setupLog.Error(
@@ -172,6 +176,21 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = mgr.AddHealthzCheck("health", ctrlrthealthz.Ping); err != nil {
+		setupLog.Error(
+			err, "unable to set up health check",
+			"aws.service", awsServiceAlias,
+		)
+		os.Exit(1)
+	}
+	if err = mgr.AddReadyzCheck("check", ctrlrthealthz.Ping); err != nil {
+		setupLog.Error(
+			err, "unable to set up ready check",
+			"aws.service", awsServiceAlias,
+		)
+		os.Exit(1)
+	}
+
 	setupLog.Info(
 		"starting manager",
 		"aws.service", awsServiceAlias,

templates/config/controller/deployment.yaml.tpl

@@ -79,6 +79,18 @@ spec:
           capabilities:
             drop:
               - ALL
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
       securityContext:
         seccompProfile:
           type: RuntimeDefault

templates/helm/templates/deployment.yaml.tpl

@@ -132,6 +132,18 @@ spec:
           capabilities:
             drop:
               - ALL
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
       securityContext:
         seccompProfile:
           type: RuntimeDefault