Ignore flapping state if flapping detection isn't enabled

Author: yhabteab

internal/icinga2/api_responses.go

@@ -137,6 +137,7 @@ type HostServiceRuntimeAttributes struct {
 	Acknowledgement           int         `json:"acknowledgement"`
 	IsFlapping                bool        `json:"flapping"`
 	AcknowledgementLastChange UnixFloat   `json:"acknowledgement_last_change"`
+	EnableFlapping            bool        `json:"enable_flapping"`
 }
 
 // ObjectQueriesResult represents the Icinga 2 API Object Queries Result wrapper object.
@@ -288,6 +289,14 @@ type ObjectCreatedDeleted struct {
 	EventType  string `json:"type"`
 }
 
+// IcingaAppStatus represents the Icinga 2 API status endpoint query result of type IcingaApplication.
+// https://icinga.com/docs/icinga-2/latest/doc/12-icinga2-api/#status-and-statistics
+type IcingaAppStatus struct {
+	App struct {
+		EnableFlapping bool `json:"enable_flapping"`
+	} `json:"app"`
+}
+
 // UnmarshalEventStreamResponse unmarshal a JSON response line from the Icinga 2 API Event Stream.
 //
 // The function expects an Icinga 2 API Event Stream Response in its JSON form and tries to unmarshal it into one of the

internal/icinga2/client.go

@@ -265,7 +265,9 @@ func (client *Client) buildAcknowledgementEvent(ctx context.Context, ack *Acknow
 		if err != nil {
 			return nil, err
 		}
-		if !isMuted(queryResult) {
+		if muted, err := isMuted(ctx, client, queryResult); err != nil {
+			return nil, err
+		} else if !muted {
 			ev.Message = queryResult.Attrs.LastCheckResult.Output
 			ev.SetMute(false, "Acknowledgement cleared")
 		}
@@ -310,7 +312,9 @@ func (client *Client) buildDowntimeEvent(ctx context.Context, d Downtime, startE
 		if err != nil {
 			return nil, err
 		}
-		if !isMuted(queryResult) {
+		if muted, err := isMuted(ctx, client, queryResult); err != nil {
+			return nil, err
+		} else if !muted {
 			// When a downtime is cancelled/expired and there's no other active downtime/ack, we're going to send some
 			// notifications if there's still an active incident. Therefore, we need the most recent CheckResult of
 			// that Checkable to use it for the notifications.
@@ -347,7 +351,9 @@ func (client *Client) buildFlappingEvent(ctx context.Context, flapping *Flapping
 		if err != nil {
 			return nil, err
 		}
-		if !isMuted(queryResult) {
+		if muted, err := isMuted(ctx, client, queryResult); err != nil {
+			return nil, err
+		} else if !muted {
 			ev.Message = queryResult.Attrs.LastCheckResult.Output
 			ev.SetMute(false, reason)
 		}

internal/icinga2/client_api.go

@@ -130,6 +130,31 @@ func (client *Client) queryObjectsApiDirect(ctx context.Context, objType, objNam
 		map[string]string{"Accept": "application/json"})
 }
 
+// fetchIcingaAppStatus retrieves the global state of the IcingaApplication type via the /v1/status endpoint.
+func (client *Client) fetchIcingaAppStatus(ctx context.Context) (*IcingaAppStatus, error) {
+	response, err := client.queryObjectsApi(
+		ctx,
+		[]string{"/v1/status/IcingaApplication/"},
+		http.MethodGet,
+		nil,
+		map[string]string{"Accept": "application/json"})
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, response)
+		_ = response.Close()
+	}()
+
+	status := new(IcingaAppStatus)
+	if err := json.NewDecoder(response).Decode(status); err != nil {
+		return nil, err
+	}
+
+	return status, nil
+}
+
 // queryObjectsApiQuery sends a query to the Icinga 2 API /v1/objects to receive data of the given objType.
 func (client *Client) queryObjectsApiQuery(ctx context.Context, objType string, query map[string]any) (io.ReadCloser, error) {
 	reqBody, err := json.Marshal(query)
@@ -260,6 +285,11 @@ func (client *Client) checkMissedChanges(ctx context.Context, objType string, ca
 		}
 
 		attrs := objQueriesResult.Attrs
+		checkableIsMuted, err := isMuted(ctx, client, &objQueriesResult)
+		if err != nil {
+			return err
+		}
+
 		var fakeEv *event.Event
 		if attrs.Acknowledgement != AcknowledgementNone {
 			ackComment, err := client.fetchAcknowledgementComment(ctx, hostName, serviceName, attrs.AcknowledgementLastChange.Time())
@@ -275,17 +305,17 @@ func (client *Client) checkMissedChanges(ctx context.Context, objType string, ca
 			if err != nil {
 				return fmt.Errorf("failed to construct Event from Acknowledgement response, %w", err)
 			}
-		} else if isMuted(&objQueriesResult) {
+		} else if checkableIsMuted {
 			fakeEv, err = client.buildCommonEvent(ctx, hostName, serviceName)
 			if err != nil {
 				return fmt.Errorf("failed to construct checkable fake mute event: %w", err)
 			}
 
 			fakeEv.Type = event.TypeMute
-			if attrs.IsFlapping {
-				fakeEv.SetMute(true, "Checkable is flapping, but we missed the Icinga 2 FlappingStart event")
-			} else {
+			if attrs.DowntimeDepth != 0 {
 				fakeEv.SetMute(true, "Checkable is in downtime, but we missed the Icinga 2 DowntimeStart event")
+			} else {
+				fakeEv.SetMute(true, "Checkable is flapping, but we missed the Icinga 2 FlappingStart event")
 			}
 		} else {
 			// This could potentially produce numerous superfluous database (event table) entries if we generate such

internal/icinga2/util.go

@@ -1,6 +1,7 @@
 package icinga2
 
 import (
+	"context"
 	"net/url"
 	"strings"
 )
@@ -18,6 +19,21 @@ func rawurlencode(s string) string {
 }
 
 // isMuted returns true if the given checkable is either in Downtime, Flapping or acknowledged, otherwise false.
-func isMuted(checkable *ObjectQueriesResult[HostServiceRuntimeAttributes]) bool {
-	return checkable.Attrs.IsFlapping || checkable.Attrs.Acknowledgement != AcknowledgementNone || checkable.Attrs.DowntimeDepth != 0
+// When the checkable is Flapping, and neither the flapping detection for that Checkable nor for the entire zone is
+// enabled, this will always return false.
+func isMuted(ctx context.Context, client *Client, checkable *ObjectQueriesResult[HostServiceRuntimeAttributes]) (bool, error) {
+	if checkable.Attrs.Acknowledgement != AcknowledgementNone || checkable.Attrs.DowntimeDepth != 0 {
+		return true, nil
+	}
+
+	if checkable.Attrs.IsFlapping && checkable.Attrs.EnableFlapping {
+		status, err := client.fetchIcingaAppStatus(ctx)
+		if err != nil {
+			return false, err
+		}
+
+		return status.App.EnableFlapping, nil
+	}
+
+	return false, nil
 }