Skip to content

Commit a9c9105

Browse files
yhabteabyhabteab
committed
Offload evaluable configs from Incident to a common place
1 parent 3e813a2 commit a9c9105

File tree

4 files changed

+471
-110
lines changed

4 files changed

+471
-110
lines changed

‎internal/config/evaluable_config.go

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package config
2+
3+
import (
4+
"github.com/icinga/icinga-notifications/internal/filter"
5+
"github.com/icinga/icinga-notifications/internal/rule"
6+
)
7+
8+
// EvalOptions specifies optional callbacks that are executed upon certain filter evaluation events.
9+
type EvalOptions[T, U any] struct {
10+
// OnPreEvaluate can be used to perform arbitrary actions before evaluating the current entry of type "T".
11+
// An entry of type "T" for which this hook returns "false" will be excluded from evaluation.
12+
OnPreEvaluate func(T) bool
13+
14+
// OnError can be used to perform arbitrary actions on filter evaluation errors.
15+
// The original filter evaluation error is passed to this function as well as the current
16+
// entry of type "T", whose filter evaluation triggered the error.
17+
//
18+
// By default, the filter evaluation doesn't get interrupted if any of them fail, instead it will continue
19+
// evaluating all the remaining entries. However, you can override this behaviour by returning "false" in
20+
// your handler, in which case the filter evaluation is aborted prematurely.
21+
OnError func(T, error) bool
22+
23+
// OnFilterMatch can be used to perform arbitrary actions after a successful filter evaluation of type "T".
24+
// This callback obtains the current entry of type "T" as an argument, whose filter matched on the filterableTest.
25+
//
26+
// Note, any error returned by the OnFilterMatch hook causes the filter evaluation to be aborted
27+
// immediately before even reaching the remaining ones.
28+
OnFilterMatch func(T) error
29+
30+
// OnAllConfigEvaluated can be used to perform some post filter evaluation actions.
31+
// This handler receives an arbitrary value, be it a result of any filter evaluation or a made-up one of type "U".
32+
//
33+
// OnAllConfigEvaluated will only be called once all the entries of type "T" are evaluated, though it doesn't
34+
// necessarily depend on the result of the individual entry filter evaluation. If the individual Eval* receivers
35+
// don't return prematurely with an error, this hook is guaranteed to be called in any other cases. However, you
36+
// should be aware, that this hook may not be supported by all Eval* methods.
37+
OnAllConfigEvaluated func(U)
38+
}
39+
40+
// Evaluable manages an evaluable config types in a centralised and structured way.
41+
// An evaluable config is a config type that allows to evaluate filter expressions in some way.
42+
type Evaluable struct {
43+
Rules map[int64]bool `db:"-"`
44+
RuleEntries map[int64]*rule.Entry `db:"-" json:"-"`
45+
}
46+
47+
// NewEvaluable returns a fully initialised and ready to use Evaluable type.
48+
func NewEvaluable() *Evaluable {
49+
return &Evaluable{
50+
Rules: make(map[int64]bool),
51+
RuleEntries: make(map[int64]*rule.Entry),
52+
}
53+
}
54+
55+
// EvaluateRules evaluates all the configured event rule.Rule(s) for the given filter.Filterable object.
56+
//
57+
// Please note that this function may not always evaluate *all* configured rules from the specified RuntimeConfig,
58+
// as it internally caches all previously matched rules based on their ID.
59+
//
60+
// EvaluateRules allows you to specify EvalOptions and hook up certain filter evaluation steps.
61+
// This function does not support the EvalOptions.OnAllConfigEvaluated callback and will never trigger
62+
// it (if provided). Please refer to the description of the individual EvalOptions to find out more about
63+
// when the hooks get triggered and possible special cases.
64+
//
65+
// Returns an error if any of the provided callbacks return an error, otherwise always nil.
66+
func (e *Evaluable) EvaluateRules(r *RuntimeConfig, filterable filter.Filterable, options EvalOptions[*rule.Rule, any]) error {
67+
for _, ru := range r.Rules {
68+
if !e.Rules[ru.ID] && (options.OnPreEvaluate == nil || options.OnPreEvaluate(ru)) {
69+
matched, err := ru.Eval(filterable)
70+
if err != nil && options.OnError != nil && !options.OnError(ru, err) {
71+
return err
72+
}
73+
if err != nil || !matched {
74+
continue
75+
}
76+
77+
if options.OnFilterMatch != nil {
78+
if err := options.OnFilterMatch(ru); err != nil {
79+
return err
80+
}
81+
}
82+
83+
e.Rules[ru.ID] = true
84+
}
85+
}
86+
87+
return nil
88+
}
89+
90+
// EvaluateRuleEntries evaluates all the configured rule.Entry for the provided filter.Filterable object.
91+
//
92+
// This function allows you to specify EvalOptions and hook up certain filter evaluation steps.
93+
// Currently, EvaluateRuleEntries fully support all the available EvalOptions. Please refer to the
94+
// description of the individual EvalOptions to find out more about when the hooks get triggered and
95+
// possible special cases.
96+
//
97+
// Returns an error if any of the provided callbacks return an error, otherwise always nil.
98+
func (e *Evaluable) EvaluateRuleEntries(r *RuntimeConfig, filterable filter.Filterable, options EvalOptions[*rule.Entry, any]) error {
99+
retryAfter := rule.RetryNever
100+
101+
for ruleID := range e.Rules {
102+
ru := r.Rules[ruleID]
103+
if ru == nil {
104+
// It would be appropriate to have a debug log here, but unfortunately we don't have access to a logger.
105+
continue
106+
}
107+
108+
for _, entry := range ru.Entries {
109+
if options.OnPreEvaluate != nil && !options.OnPreEvaluate(entry) {
110+
continue
111+
}
112+
113+
if matched, err := entry.Eval(filterable); err != nil {
114+
if options.OnError != nil && !options.OnError(entry, err) {
115+
return err
116+
}
117+
} else if cond, ok := filterable.(*rule.EscalationFilter); !matched && ok {
118+
incidentAgeFilter := cond.ReevaluateAfter(entry.Condition)
119+
retryAfter = min(retryAfter, incidentAgeFilter)
120+
} else if matched {
121+
if options.OnFilterMatch != nil {
122+
if err := options.OnFilterMatch(entry); err != nil {
123+
return err
124+
}
125+
}
126+
127+
e.RuleEntries[entry.ID] = entry
128+
}
129+
}
130+
}
131+
132+
if options.OnAllConfigEvaluated != nil {
133+
options.OnAllConfigEvaluated(retryAfter)
134+
}
135+
136+
return nil
137+
}

‎internal/config/evaluable_config_test.go

+243
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
package config
2+
3+
import (
4+
"fmt"
5+
"github.com/icinga/icinga-notifications/internal/filter"
6+
"github.com/icinga/icinga-notifications/internal/rule"
7+
"github.com/stretchr/testify/require"
8+
"maps"
9+
"testing"
10+
"time"
11+
)
12+
13+
const defaultDivisor = 3
14+
15+
func TestEvaluableConfig(t *testing.T) {
16+
t.Parallel()
17+
18+
runtimeConfigTest := new(RuntimeConfig)
19+
runtimeConfigTest.Rules = make(map[int64]*rule.Rule)
20+
for i := 1; i <= 50; i++ {
21+
runtimeConfigTest.Rules[int64(i)] = makeRule(t, i)
22+
}
23+
24+
t.Run("NewEvaluable", func(t *testing.T) {
25+
t.Parallel()
26+
27+
e := NewEvaluable()
28+
require.NotNil(t, e, "it should create a fully initialised evaluable config")
29+
require.NotNil(t, e.Rules)
30+
require.NotNil(t, e.RuleEntries)
31+
})
32+
33+
t.Run("EvaluateRules", func(t *testing.T) {
34+
t.Parallel()
35+
36+
runtime := new(RuntimeConfig)
37+
runtime.Rules = maps.Clone(runtimeConfigTest.Rules)
38+
39+
expectedLen := len(runtime.Rules) / defaultDivisor
40+
options := EvalOptions[*rule.Rule, any]{}
41+
e := NewEvaluable()
42+
assertRules := func(expectedLen *int, expectError bool) {
43+
if expectError {
44+
require.Error(t, e.EvaluateRules(runtime, new(filterableTest), options))
45+
} else {
46+
require.NoError(t, e.EvaluateRules(runtime, new(filterableTest), options))
47+
}
48+
require.Len(t, e.Rules, *expectedLen)
49+
}
50+
51+
assertRules(&expectedLen, false)
52+
maps.DeleteFunc(e.Rules, func(ruleID int64, _ bool) bool { return int(ruleID) > expectedLen/2 })
53+
54+
options.OnPreEvaluate = func(r *rule.Rule) bool {
55+
require.Falsef(t, e.Rules[r.ID], "EvaluateRules() shouldn't evaluate %q twice", r.Name)
56+
return true
57+
}
58+
options.OnError = func(r *rule.Rule, err error) bool {
59+
require.EqualError(t, err, `"nonexistent" is not a valid filter key`)
60+
require.Truef(t, r.ID%defaultDivisor != 0, "evaluating rule %q should not fail", r.Name)
61+
return true
62+
}
63+
options.OnFilterMatch = func(r *rule.Rule) error {
64+
require.Falsef(t, e.Rules[r.ID], "EvaluateRules() shouldn't evaluate %q twice", r.Name)
65+
return nil
66+
}
67+
68+
assertRules(&expectedLen, false)
69+
maps.DeleteFunc(e.Rules, func(ruleID int64, _ bool) bool { return int(ruleID) > expectedLen/2 })
70+
71+
lenBeforeError := new(int)
72+
options.OnError = func(r *rule.Rule, err error) bool {
73+
if *lenBeforeError != 0 {
74+
require.Fail(t, "OnError() shouldn't have been called again")
75+
}
76+
77+
require.EqualError(t, err, `"nonexistent" is not a valid filter key`)
78+
require.Truef(t, r.ID%defaultDivisor != 0, "evaluating rule %q should not fail", r.Name)
79+
80+
*lenBeforeError = len(e.Rules)
81+
return false // This should let the evaluation fail completely!
82+
}
83+
assertRules(lenBeforeError, true)
84+
maps.DeleteFunc(e.Rules, func(ruleID int64, _ bool) bool { return int(ruleID) > expectedLen/2 })
85+
86+
*lenBeforeError = 0
87+
options.OnError = nil
88+
options.OnFilterMatch = func(r *rule.Rule) error {
89+
if *lenBeforeError != 0 {
90+
require.Fail(t, "OnFilterMatch() shouldn't have been called again")
91+
}
92+
93+
*lenBeforeError = len(e.Rules)
94+
return fmt.Errorf("OnFilterMatch() failed badly") // This should let the evaluation fail completely!
95+
}
96+
assertRules(lenBeforeError, true)
97+
})
98+
99+
t.Run("EvaluateRuleEntries", func(t *testing.T) {
100+
t.Parallel()
101+
102+
runtime := new(RuntimeConfig)
103+
runtime.Rules = maps.Clone(runtimeConfigTest.Rules)
104+
105+
e := NewEvaluable()
106+
options := EvalOptions[*rule.Entry, any]{}
107+
108+
expectedLen := 0
109+
filterContext := &rule.EscalationFilter{IncidentSeverity: 9} // Event severity "emergency"
110+
assertEntries := func(expectedLen *int, expectError bool) {
111+
if expectError {
112+
require.Error(t, e.EvaluateRuleEntries(runtime, filterContext, options))
113+
} else {
114+
require.NoError(t, e.EvaluateRuleEntries(runtime, filterContext, options))
115+
}
116+
require.Len(t, e.RuleEntries, *expectedLen)
117+
e.RuleEntries = make(map[int64]*rule.Entry)
118+
}
119+
120+
assertEntries(&expectedLen, false)
121+
require.NoError(t, e.EvaluateRules(runtime, new(filterableTest), EvalOptions[*rule.Rule, any]{}))
122+
require.Len(t, e.Rules, len(runtime.Rules)/defaultDivisor)
123+
expectedLen = len(runtime.Rules)/defaultDivisor - 5 // 15/3 => (5) valid entries are going to be deleted below.
124+
125+
// Drop some random rules from the runtime config to simulate a runtime config deletion!
126+
maps.DeleteFunc(runtime.Rules, func(ruleID int64, _ *rule.Rule) bool { return ruleID > 35 && ruleID%defaultDivisor == 0 })
127+
128+
options.OnPreEvaluate = func(re *rule.Entry) bool {
129+
if re.RuleID > 35 && re.RuleID%defaultDivisor == 0 { // Those rules are deleted from our runtime config.
130+
require.Failf(t, "OnPreEvaluate() shouldn't have been called", "rule %d was deleted from runtime config", re.RuleID)
131+
}
132+
133+
require.Nilf(t, e.RuleEntries[re.ID], "EvaluateRuleEntries() shouldn't evaluate entry %d twice", re.ID)
134+
return true
135+
}
136+
options.OnError = func(re *rule.Entry, err error) bool {
137+
require.EqualError(t, err, `unknown severity "evaluable"`)
138+
require.Truef(t, re.RuleID%defaultDivisor == 0, "evaluating rule entry %d should not fail", re.ID)
139+
return true
140+
}
141+
options.OnFilterMatch = func(re *rule.Entry) error {
142+
require.Nilf(t, e.RuleEntries[re.ID], "OnPreEvaluate() shouldn't evaluate %d twice", re.ID)
143+
return nil
144+
}
145+
assertEntries(&expectedLen, false)
146+
147+
lenBeforeError := new(int)
148+
options.OnError = func(re *rule.Entry, err error) bool {
149+
if *lenBeforeError != 0 {
150+
require.Fail(t, "OnError() shouldn't have been called again")
151+
}
152+
153+
require.EqualError(t, err, `unknown severity "evaluable"`)
154+
require.Truef(t, re.RuleID%defaultDivisor == 0, "evaluating rule entry %d should not fail", re.ID)
155+
156+
*lenBeforeError = len(e.RuleEntries)
157+
return false // This should let the evaluation fail completely!
158+
}
159+
assertEntries(lenBeforeError, true)
160+
161+
*lenBeforeError = 0
162+
options.OnError = nil
163+
options.OnFilterMatch = func(re *rule.Entry) error {
164+
if *lenBeforeError != 0 {
165+
require.Fail(t, "OnFilterMatch() shouldn't have been called again")
166+
}
167+
168+
*lenBeforeError = len(e.RuleEntries)
169+
return fmt.Errorf("OnFilterMatch() failed badly") // This should let the evaluation fail completely!
170+
}
171+
assertEntries(lenBeforeError, true)
172+
173+
expectedLen = 0
174+
filterContext.IncidentSeverity = 1 // OK
175+
filterContext.IncidentAge = 5 * time.Minute
176+
177+
options.OnFilterMatch = nil
178+
options.OnPreEvaluate = func(re *rule.Entry) bool { return re.RuleID < 5 }
179+
options.OnAllConfigEvaluated = func(result any) {
180+
retryAfter := result.(time.Duration)
181+
// The filter string of the escalation condition is incident_age>=10m and the actual incident age is 5m.
182+
require.Equal(t, 5*time.Minute, retryAfter)
183+
}
184+
assertEntries(&expectedLen, false)
185+
})
186+
}
187+
188+
func makeRule(t *testing.T, i int) *rule.Rule {
189+
r := new(rule.Rule)
190+
r.ID = int64(i)
191+
r.Name = fmt.Sprintf("rule-%d", i)
192+
r.Entries = make(map[int64]*rule.Entry)
193+
194+
invalidSeverity, err := filter.Parse("incident_severity=evaluable")
195+
require.NoError(t, err, "parsing incident_severity=evaluable shouldn't fail")
196+
197+
redundant := new(rule.Entry)
198+
redundant.ID = r.ID * 150 // It must be large enough to avoid colliding with others!
199+
redundant.RuleID = r.ID
200+
redundant.Condition = invalidSeverity
201+
202+
nonexistent, err := filter.Parse("nonexistent=evaluable")
203+
require.NoError(t, err, "parsing nonexistent=evaluable shouldn't fail")
204+
205+
r.Entries[redundant.ID] = redundant
206+
r.ObjectFilter = nonexistent
207+
if i%defaultDivisor == 0 {
208+
objCond, err := filter.Parse("host=evaluable")
209+
require.NoError(t, err, "parsing host=evaluable shouldn't fail")
210+
211+
escalationCond, err := filter.Parse("incident_severity>warning||incident_age>=10m")
212+
require.NoError(t, err, "parsing incident_severity>=ok shouldn't fail")
213+
214+
entry := new(rule.Entry)
215+
entry.ID = r.ID * 2
216+
entry.RuleID = r.ID
217+
entry.Condition = escalationCond
218+
219+
r.ObjectFilter = objCond
220+
r.Entries[entry.ID] = entry
221+
}
222+
223+
return r
224+
}
225+
226+
// filterableTest is a test type that simulates a filter evaluation and eliminates
227+
// the need of having to import e.g. the object package.
228+
type filterableTest struct{}
229+
230+
func (f *filterableTest) EvalEqual(k string, v string) (bool, error) {
231+
if k != "host" {
232+
return false, fmt.Errorf("%q is not a valid filter key", k)
233+
}
234+
235+
return v == "evaluable", nil
236+
}
237+
238+
func (f *filterableTest) EvalExists(_ string) bool { return true }
239+
func (f *filterableTest) EvalLess(_ string, _ string) (bool, error) {
240+
panic("Oh dear - you shouldn't have called me")
241+
}
242+
func (f *filterableTest) EvalLike(_, _ string) (bool, error) { return f.EvalLess("", "") }
243+
func (f *filterableTest) EvalLessOrEqual(_, _ string) (bool, error) { return f.EvalLess("", "") }

0 commit comments

Comments
 (0)