[detector] Add threshold-based analysis settings and detection logic

Author: Litemn

pkg/degradation-detector/analysisSettings.go

@@ -8,6 +8,23 @@ const (
 	ImprovementEvent
 )
 
+// AnalysisKind selects which algorithm to use for detection.
+// Default (zero) keeps backward-compatible change-point detection.
+type AnalysisKind int
+
+const (
+	ChangePointAnalysis AnalysisKind = iota
+	ThresholdAnalysis
+)
+
+// ThresholdMode specifies how the latest value should be compared to the threshold.
+type ThresholdMode int
+
+const (
+	ThresholdGreaterThan ThresholdMode = iota
+	ThresholdLessThan
+)
+
 type AnalysisSettings struct {
 	ReportType ReportType
 	// Determines the minimum length of a segment the larger the segment the more accurate the analysis but it will take more time to detect degradation
@@ -23,6 +40,10 @@ type AnalysisSettings struct {
 	// Number of days to check for missing data.
 	// The default value is -3 (3 days ago).
 	DaysToCheckMissing int
+
+	AnalysisKind   AnalysisKind
+	ThresholdMode  ThresholdMode
+	ThresholdValue float64
 }
 
 func (s AnalysisSettings) GetReportType() ReportType {
@@ -47,3 +68,12 @@ func (s AnalysisSettings) GetDaysToCheckMissing() int {
 	}
 	return s.DaysToCheckMissing
 }
+
+// GetAnalysisKind returns the analysis kind setting.
+func (s AnalysisSettings) GetAnalysisKind() AnalysisKind { return s.AnalysisKind }
+
+// GetThresholdMode returns the threshold comparison mode.
+func (s AnalysisSettings) GetThresholdMode() ThresholdMode { return s.ThresholdMode }
+
+// GetThresholdValue returns the threshold value.
+func (s AnalysisSettings) GetThresholdValue() float64 { return s.ThresholdValue }

pkg/degradation-detector/degradationDetector.go

@@ -26,6 +26,10 @@ type analysisSettings interface {
 	GetMedianDifferenceThreshold() float64
 	GetEffectSizeThreshold() float64
 	GetDaysToCheckMissing() int
+
+	GetAnalysisKind() AnalysisKind
+	GetThresholdMode() ThresholdMode
+	GetThresholdValue() float64
 }
 
 func (v CenterValues) PercentageChange() float64 {
@@ -35,6 +39,10 @@ func (v CenterValues) PercentageChange() float64 {
 func detectDegradations(values []int, builds []string, timestamps []int64, analysisSettings analysisSettings) []Degradation {
 	degradations := make([]Degradation, 0)
 
+	if analysisSettings.GetAnalysisKind() == ThresholdAnalysis {
+		return detectThresholdExceed(values, builds, timestamps, analysisSettings)
+	}
+
 	minimumSegmentLength := analysisSettings.GetMinimumSegmentLength()
 	if minimumSegmentLength == 0 {
 		minimumSegmentLength = 5
@@ -115,6 +123,44 @@ func detectDegradations(values []int, builds []string, timestamps []int64, analy
 	return degradations
 }
 
+// detectThresholdExceed emits a degradation when the latest value crosses the configured threshold
+// according to the selected ThresholdMode. For GreaterThan mode, strictly greater (>) is used; for
+// LessThan mode, strictly less (<) is used.
+func detectThresholdExceed(values []int, builds []string, timestamps []int64, s analysisSettings) []Degradation {
+	result := make([]Degradation, 0)
+	if len(values) == 0 || len(builds) == 0 || len(timestamps) == 0 {
+		return result
+	}
+	lastIdx := len(values) - 1
+	last := float64(values[lastIdx])
+	threshold := s.GetThresholdValue()
+
+	meets := false
+	switch s.GetThresholdMode() {
+	case ThresholdGreaterThan:
+		meets = last > threshold
+	case ThresholdLessThan:
+		meets = last < threshold
+	}
+	if !meets {
+		return result
+	}
+	// Treat exceeding threshold as degradation event; consumer can filter by ReportType if needed
+	var previous float64
+	if lastIdx > 0 {
+		previous = float64(values[lastIdx-1])
+	} else {
+		previous = threshold
+	}
+	result = append(result, Degradation{
+		Build:         builds[lastIdx],
+		timestamp:     timestamps[lastIdx],
+		medianValues:  CenterValues{previousValue: previous, newValue: last},
+		IsDegradation: true,
+	})
+	return result
+}
+
 func GetSegmentsBetweenChangePoints(changePoints []int, values []int) [][]int {
 	segments := make([][]int, 0, len(changePoints)+1)
 	prevChangePoint := 0

pkg/degradation-detector/setting/aiaTestTokenSettings.go

@@ -0,0 +1,38 @@
+package setting
+
+import detector "github.com/JetBrains/ij-perf-report-aggregator/pkg/degradation-detector"
+
+func GenerateAIATestTokenSettings() []detector.PerformanceSettings {
+	metrics := map[string][]string{
+		"stage: tokenQuota": {"currentPercents"},
+		"prod: tokenQuota":  {"currentPercents"},
+	}
+
+	settings := make([]detector.PerformanceSettings, 0, 10)
+
+	for test, metrics := range metrics {
+		for _, metric := range metrics {
+			settings = append(settings, detector.PerformanceSettings{
+				Db:      "perfintDev",
+				Table:   "ml",
+				Project: test,
+				BaseSettings: detector.BaseSettings{
+					Machine: "intellij-linux-%-aws-%",
+					Metric:  metric,
+					Branch:  "master",
+					SlackSettings: detector.SlackSettings{
+						Channel:     "ai-assistant-autotest-notifications",
+						ProductLink: "ml/dev",
+					},
+					AnalysisSettings: detector.AnalysisSettings{
+						ReportType:     detector.AllEvent,
+						AnalysisKind:   detector.ThresholdAnalysis,
+						ThresholdMode:  detector.ThresholdGreaterThan,
+						ThresholdValue: 95,
+					},
+				},
+			})
+		}
+	}
+	return settings
+}

pkg/degradation-detector/setting/setting_test.go

@@ -24,3 +24,14 @@ func TestMavenSetting(t *testing.T) {
 		assert.Equal(t, detector.DegradationEvent, setting.AnalysisSettings.ReportType)
 	}
 }
+
+func TestGenerateAIATestTokenSettings(t *testing.T) {
+	t.Parallel()
+	settings := make([]detector.PerformanceSettings, 0, 1000)
+	settings = append(settings, GenerateAIATestTokenSettings()...)
+	for _, setting := range settings {
+		assert.Equal(t, detector.ThresholdAnalysis, setting.AnalysisSettings.AnalysisKind)
+		assert.Equal(t, detector.ThresholdGreaterThan, setting.AnalysisSettings.ThresholdMode)
+		assert.InEpsilon(t, 95.0, setting.AnalysisSettings.ThresholdValue, 0.0001)
+	}
+}

pkg/degradation-detector/threshold_analysis_test.go

@@ -0,0 +1,99 @@
+package degradation_detector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestThresholdGreaterThan_TriggersWhenStrictlyGreater(t *testing.T) {
+	t.Parallel()
+	values := []int{90, 96}
+	builds := []string{"b0", "b1"}
+	times := []int64{0, 123}
+
+	settings := AnalysisSettings{
+		AnalysisKind:   ThresholdAnalysis,
+		ThresholdMode:  ThresholdGreaterThan,
+		ThresholdValue: 95,
+	}
+
+	degradations := detectDegradations(values, builds, times, settings)
+	assert.Len(t, degradations, 1)
+	deg := degradations[0]
+	assert.Equal(t, "b1", deg.Build)
+	assert.Equal(t, int64(123), deg.timestamp)
+	assert.True(t, deg.IsDegradation)
+	assert.InEpsilon(t, 90.0, deg.medianValues.previousValue, 0.0001)
+	assert.InEpsilon(t, 96.0, deg.medianValues.newValue, 0.0001)
+}
+
+func TestThresholdGreaterThan_DoesNotTriggerOnEqual(t *testing.T) {
+	t.Parallel()
+	values := []int{95}
+	builds := []string{"b0"}
+	times := []int64{0}
+
+	settings := AnalysisSettings{
+		AnalysisKind:   ThresholdAnalysis,
+		ThresholdMode:  ThresholdGreaterThan,
+		ThresholdValue: 95,
+	}
+
+	degradations := detectDegradations(values, builds, times, settings)
+	assert.Empty(t, degradations)
+}
+
+func TestThresholdLessThan_TriggersWhenStrictlyLess(t *testing.T) {
+	t.Parallel()
+	values := []int{100, 80}
+	builds := []string{"b0", "b1"}
+	times := []int64{0, 77}
+
+	settings := AnalysisSettings{
+		AnalysisKind:   ThresholdAnalysis,
+		ThresholdMode:  ThresholdLessThan,
+		ThresholdValue: 85,
+	}
+
+	degradations := detectDegradations(values, builds, times, settings)
+	assert.Len(t, degradations, 1)
+	deg := degradations[0]
+	assert.Equal(t, "b1", deg.Build)
+	assert.Equal(t, int64(77), deg.timestamp)
+	assert.True(t, deg.IsDegradation)
+	assert.InEpsilon(t, 100.0, deg.medianValues.previousValue, 0.0001)
+	assert.InEpsilon(t, 80.0, deg.medianValues.newValue, 0.0001)
+}
+
+func TestThresholdLessThan_DoesNotTriggerOnEqual(t *testing.T) {
+	t.Parallel()
+	values := []int{85}
+	builds := []string{"b0"}
+	times := []int64{0}
+
+	settings := AnalysisSettings{
+		AnalysisKind:   ThresholdAnalysis,
+		ThresholdMode:  ThresholdLessThan,
+		ThresholdValue: 85,
+	}
+
+	degradations := detectDegradations(values, builds, times, settings)
+	assert.Empty(t, degradations)
+}
+
+func TestThreshold_NoDataReturnsEmpty(t *testing.T) {
+	t.Parallel()
+	var values []int
+	var builds []string
+	var times []int64
+
+	settings := AnalysisSettings{
+		AnalysisKind:   ThresholdAnalysis,
+		ThresholdMode:  ThresholdGreaterThan,
+		ThresholdValue: 95,
+	}
+
+	degradations := detectDegradations(values, builds, times, settings)
+	assert.Empty(t, degradations)
+}