[health] improve health reporting

Author: capcom6

api/requests.http

@@ -4,10 +4,16 @@
 @phone={{$dotenv PHONE}}
 
 ###
-GET {{baseUrl}}/health HTTP/1.1
+GET {{baseUrl}}/3rdparty/v1/health HTTP/1.1
 
 ###
-GET {{baseUrl}}/3rdparty/v1/health HTTP/1.1
+GET {{baseUrl}}/3rdparty/v1/health/startup HTTP/1.1
+
+###
+GET {{baseUrl}}/3rdparty/v1/health/ready HTTP/1.1
+
+###
+GET {{baseUrl}}/3rdparty/v1/health/live HTTP/1.1
 
 ###
 POST {{baseUrl}}/3rdparty/v1/messages?skipPhoneValidation=false&deviceActiveWithin=240 HTTP/1.1
@@ -189,3 +195,12 @@ Content-Type: application/json
 ###
 GET http://localhost:3000/metrics HTTP/1.1
 
+###
+GET http://localhost:3000/health/startup HTTP/1.1
+
+###
+GET http://localhost:3000/health/ready HTTP/1.1
+
+###
+GET http://localhost:3000/health/live HTTP/1.1
+

build/package/Dockerfile

@@ -48,6 +48,6 @@ USER guest
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
-    CMD curl -fs http://localhost:3000/health
+    CMD curl -fs http://localhost:3000/health/live
 
 CMD [ "/app/app" ]

deployments/helm-chart/templates/deployment.yaml

@@ -92,21 +92,31 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           livenessProbe:
+            httpGet:
+              path: /health/live
+              port: http
             initialDelaySeconds: 30
             periodSeconds: 10
             timeoutSeconds: 5
             failureThreshold: 3
+
+          readinessProbe:
             httpGet:
-              path: /health
+              path: /health/ready
               port: http
-          readinessProbe:
             initialDelaySeconds: 10
             periodSeconds: 5
             timeoutSeconds: 3
             failureThreshold: 3
+
+          startupProbe:
             httpGet:
-              path: /health
+              path: /health/startup
               port: http
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
           {{- if .Values.gateway.config.enabled }}
           volumeMounts:
             - name: config-volume

internal/sms-gateway/app.go

@@ -37,7 +37,7 @@ var Module = fx.Module(
 	"server",
 	logger.Module,
 	appconfig.Module,
-	appdb.Module,
+	appdb.Module(),
 	http.Module,
 	validator.Module,
 	openapi.Module(),
@@ -49,7 +49,7 @@ var Module = fx.Module(
 	pubsub.Module(),
 	events.Module,
 	messages.Module(),
-	health.Module,
+	health.Module(),
 	webhooks.Module,
 	settings.Module,
 	devices.Module,

internal/sms-gateway/handlers/health.go

@@ -5,75 +5,102 @@ import (
 	"github.com/android-sms-gateway/server/internal/sms-gateway/handlers/base"
 	"github.com/android-sms-gateway/server/internal/sms-gateway/modules/health"
 	"github.com/android-sms-gateway/server/internal/version"
-	"github.com/capcom6/go-helpers/maps"
+	"github.com/go-playground/validator/v10"
 	"github.com/gofiber/fiber/v2"
-	"go.uber.org/fx"
+	"github.com/samber/lo"
 	"go.uber.org/zap"
 )
 
-type healthHanlderParams struct {
-	fx.In
-
-	HealthSvc *health.Service
-
-	Logger *zap.Logger
-}
-
 type healthHandler struct {
 	base.Handler
 
 	healthSvc *health.Service
+}
 
-	logger *zap.Logger
+func newHealthHandler(
+	healthSvc *health.Service,
+	logger *zap.Logger,
+	validator *validator.Validate,
+) *healthHandler {
+	return &healthHandler{
+		Handler: base.Handler{
+			Logger:    logger,
+			Validator: validator,
+		},
+		healthSvc: healthSvc,
+	}
 }
 
-//	@Summary		Health check
-//	@Description	Checks if service is healthy
+//	@Summary		Liveness probe
+//	@Description	Checks if service is running (liveness probe)
 //	@Tags			System
 //	@Produce		json
-//	@Success		200	{object}	smsgateway.HealthResponse	"Health check result"
-//	@Failure		500	{object}	smsgateway.HealthResponse	"Service is unhealthy"
-//	@Router			/3rdparty/v1/health [get]
+//	@Success		200	{object}	smsgateway.HealthResponse	"Service is alive"
+//	@Failure		503	{object}	smsgateway.HealthResponse	"Service is not alive"
+//	@Router			/health/live [get]
+//
+// Liveness probe
+func (h *healthHandler) getLiveness(c *fiber.Ctx) error {
+	return writeProbe(c, h.healthSvc.CheckLiveness(c.Context()))
+}
+
+// @Summary		Readiness probe
+// @Description	Checks if service is ready to serve traffic (readiness probe)
+// @Tags			System
+// @Produce		json
+// @Success		200	{object}	smsgateway.HealthResponse	"Service is ready"
+// @Failure		503	{object}	smsgateway.HealthResponse	"Service is not ready"
+// @Router			/health/ready [get]
+// @Router			/3rdparty/v1/health [get]
 //
-// Health check
-func (h *healthHandler) getHealth(c *fiber.Ctx) error {
-	check, err := h.healthSvc.HealthCheck(c.Context())
-	if err != nil {
-		return err
+// Readiness probe
+func (h *healthHandler) getReadiness(c *fiber.Ctx) error {
+	return writeProbe(c, h.healthSvc.CheckReadiness(c.Context()))
+}
+
+// @Summary		Startup probe
+// @Description	Checks if service has completed initialization (startup probe)
+// @Tags			System
+// @Produce		json
+// @Success		200	{object}	smsgateway.HealthResponse	"Service has completed initialization"
+// @Failure		503	{object}	smsgateway.HealthResponse	"Service has not completed initialization"
+// @Router			/health/startup [get]
+//
+// Startup probe
+func (h *healthHandler) getStartup(c *fiber.Ctx) error {
+	return writeProbe(c, h.healthSvc.CheckStartup(c.Context()))
+}
+
+func writeProbe(c *fiber.Ctx, r health.CheckResult) error {
+	status := fiber.StatusOK
+	if r.Status() == health.StatusFail {
+		status = fiber.StatusServiceUnavailable
 	}
+	return c.Status(status).JSON(makeResponse(r))
+}
 
-	res := smsgateway.HealthResponse{
-		Status:    smsgateway.HealthStatus(check.Status),
+func makeResponse(result health.CheckResult) smsgateway.HealthResponse {
+	return smsgateway.HealthResponse{
+		Status:    smsgateway.HealthStatus(result.Status()),
 		Version:   version.AppVersion,
 		ReleaseID: version.AppReleaseID(),
-		Checks: maps.MapValues(
-			check.Checks,
-			func(c health.CheckDetail) smsgateway.HealthCheck {
+		Checks: lo.MapValues(
+			result.Checks,
+			func(value health.CheckDetail, key string) smsgateway.HealthCheck {
 				return smsgateway.HealthCheck{
-					Description:   c.Description,
-					ObservedUnit:  c.ObservedUnit,
-					ObservedValue: c.ObservedValue,
-					Status:        smsgateway.HealthStatus(c.Status),
+					Description:   value.Description,
+					ObservedUnit:  value.ObservedUnit,
+					ObservedValue: value.ObservedValue,
+					Status:        smsgateway.HealthStatus(value.Status),
 				}
 			},
 		),
 	}
-
-	if check.Status == health.StatusFail {
-		return c.Status(fiber.StatusInternalServerError).JSON(res)
-	}
-
-	return c.Status(fiber.StatusOK).JSON(res)
 }
 
 func (h *healthHandler) Register(router fiber.Router) {
-	router.Get("/health", h.getHealth)
-}
-
-func newHealthHandler(params healthHanlderParams) *healthHandler {
-	return &healthHandler{
-		Handler:   base.Handler{Logger: params.Logger.Named("HealthHandler"), Validator: nil},
-		healthSvc: params.HealthSvc,
-		logger:    params.Logger,
-	}
+	router.Get("/health", h.getReadiness)
+	router.Get("/health/live", h.getLiveness)
+	router.Get("/health/ready", h.getReadiness)
+	router.Get("/health/startup", h.getStartup)
 }

internal/sms-gateway/modules/db/health.go

@@ -0,0 +1,60 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"sync/atomic"
+
+	healthmod "github.com/android-sms-gateway/server/internal/sms-gateway/modules/health"
+)
+
+type health struct {
+	db *sql.DB
+
+	failedPings atomic.Int64
+}
+
+func newHealth(db *sql.DB) *health {
+	return &health{
+		db: db,
+	}
+}
+
+// Name implements HealthProvider.
+func (h *health) Name() string {
+	return "db"
+}
+
+// LiveProbe implements HealthProvider.
+func (h *health) LiveProbe(ctx context.Context) (healthmod.Checks, error) {
+	return nil, nil
+}
+
+// ReadyProbe implements HealthProvider.
+func (h *health) ReadyProbe(ctx context.Context) (healthmod.Checks, error) {
+	pingCheck := healthmod.CheckDetail{
+		Description:   "Database ping",
+		ObservedUnit:  "failed pings",
+		ObservedValue: 0,
+		Status:        healthmod.StatusPass,
+	}
+
+	if err := h.db.PingContext(ctx); err != nil {
+		h.failedPings.Add(1)
+
+		pingCheck.Status = healthmod.StatusFail
+	} else {
+		h.failedPings.Store(0)
+	}
+
+	pingCheck.ObservedValue = int(h.failedPings.Load())
+
+	return healthmod.Checks{"ping": pingCheck}, nil
+}
+
+// StartedProbe implements HealthProvider.
+func (h *health) StartedProbe(ctx context.Context) (healthmod.Checks, error) {
+	return nil, nil
+}
+
+var _ healthmod.HealthProvider = (*health)(nil)

internal/sms-gateway/modules/db/module.go

@@ -3,13 +3,20 @@ package db
 import (
 	"github.com/jaevor/go-nanoid"
 	"go.uber.org/fx"
+
+	healthmod "github.com/android-sms-gateway/server/internal/sms-gateway/modules/health"
 )
 
 type IDGen func() string
 
-var Module = fx.Module(
-	"db",
-	fx.Provide(func() (IDGen, error) {
-		return nanoid.Standard(21)
-	}),
-)
+func Module() fx.Option {
+	return fx.Module(
+		"db",
+		fx.Provide(
+			healthmod.AsHealthProvider(newHealth),
+		),
+		fx.Provide(func() (IDGen, error) {
+			return nanoid.Standard(21)
+		}),
+	)
+}