Ignore state change events while in downtime/flapping state

Author: yhabteab

Diff for: internal/icinga2/api_responses.go

@@ -132,6 +132,7 @@ type HostServiceRuntimeAttributes struct {
 	Groups                    []string    `json:"groups"`
 	State                     int         `json:"state"`
 	StateType                 int         `json:"state_type"`
+	IsFlapping                bool        `json:"flapping"`
 	LastCheckResult           CheckResult `json:"last_check_result"`
 	LastStateChange           UnixFloat   `json:"last_state_change"`
 	DowntimeDepth             int         `json:"downtime_depth"`

Diff for: internal/icinga2/client_api.go

@@ -230,6 +230,16 @@ func (client *Client) checkMissedChanges(ctx context.Context, objType string, ca
 			return fmt.Errorf("querying API delivered a wrong object type %q", objQueriesResult.Type)
 		}
 
+		// Downtime suppresses all kind of events on that checkable, so skip it if it's in downtime.
+		if objQueriesResult.Attrs.DowntimeDepth > 0 {
+			continue
+		}
+
+		// If the checkable is not in downtime but in flapping state, all other states/events become irrelevant as well.
+		if objQueriesResult.Attrs.IsFlapping {
+			continue
+		}
+
 		// Only process HARD states
 		if objQueriesResult.Attrs.StateType == StateTypeSoft {
 			client.Logger.Debugf("Skipping SOFT event, %#v", objQueriesResult.Attrs)
@@ -428,7 +438,20 @@ func (client *Client) listenEventStream() error {
 		)
 		switch respT := resp.(type) {
 		case *StateChange:
-			// Only process HARD states
+			// Ignore any state changes while in downtime
+			if respT.DowntimeDepth > 0 {
+				stateType := "HARD"
+				if respT.StateType == StateTypeSoft {
+					stateType = "SOFT"
+				}
+
+				client.Logger.Debugf("Skipping %q state change while in downtime, %#v", stateType, respT)
+				continue
+			}
+
+			// Only process HARD states.
+			// The checkable might also be in the flapping state, but Icinga 2 doesn't expose "is_flapping" to the
+			// event streams for state change. So, the only thing we can do here is to ignore soft state changes.
 			if respT.StateType == StateTypeSoft {
 				client.Logger.Debugf("Skipping SOFT State Change, %#v", respT)
 				continue