Skip to content

Commit 022f109

Browse files
EmanueleMinottoEmanueleMinotto
committed
Add quarantine report generation and emission
Implements GitHub issue #3: generate and emit quarantine reports showing: - Number of currently disabled tests - Tests due to expire this week - Count of recently reenabled tests - Total quarantine-days debt (sum of days for all disabled tests) Reports emitted to: - GITHUB_STEP_SUMMARY (GitHub Actions job summary) - skipper-report.json (local JSON file) API: New public method Report() in all test framework packages: - testing.Report() — called automatically in finalize, or manually - testify.SkipperSuite.Report() — called manually from user code - ginkgo.Report() — called manually after RunSpecs Core: New GenerateReport() and WriteReport() functions compute metrics from the resolver and emit formatted output to file and GitHub Actions.
1 parent 95a8bd1 commit 022f109

5 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎core/report.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package core
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
"time"
8+
)
9+
10+
// Report contains metrics about disabled tests and quarantine debt.
11+
type Report struct {
12+
DisabledCount int `json:"disabled_count"`
13+
DueThisWeek []string `json:"due_this_week"`
14+
ReenabledCount int `json:"reenabled_count"`
15+
QuarantineDaysDebt int `json:"quarantine_days_debt"`
16+
GeneratedAt time.Time `json:"generated_at"`
17+
}
18+
19+
// GenerateReport generates a quarantine report from the current resolver state.
20+
// It calculates:
21+
// - disabled_count: number of tests currently disabled (disabledUntil is in future)
22+
// - due_this_week: test IDs with disabledUntil dates within next 7 days
23+
// - reenabled_count: tests that were disabled but are now enabled (disabledUntil in past)
24+
// - quarantine_days_debt: sum of days between disabledUntil and now for disabled tests
25+
func GenerateReport(resolver *SkipperResolver) *Report {
26+
if resolver == nil {
27+
return &Report{GeneratedAt: time.Now().UTC()}
28+
}
29+
30+
now := time.Now().UTC()
31+
weekFromNow := now.AddDate(0, 0, 7)
32+
33+
disabledCount := 0
34+
reenabledCount := 0
35+
dueThisWeek := []string{}
36+
quarantineDaysDebt := 0
37+
38+
for testID, disabledUntil := range resolver.cache {
39+
if disabledUntil == nil {
40+
continue
41+
}
42+
43+
if now.Before(*disabledUntil) {
44+
// Test is currently disabled
45+
disabledCount++
46+
47+
// Check if due this week
48+
if disabledUntil.Before(weekFromNow) || disabledUntil.Equal(weekFromNow) {
49+
dueThisWeek = append(dueThisWeek, testID)
50+
}
51+
52+
// Add to debt: days from now until disabledUntil
53+
days := int(disabledUntil.Sub(now).Hours() / 24)
54+
quarantineDaysDebt += days
55+
} else {
56+
// Test is reenabled (was disabled but date passed)
57+
reenabledCount++
58+
}
59+
}
60+
61+
return &Report{
62+
DisabledCount: disabledCount,
63+
DueThisWeek: dueThisWeek,
64+
ReenabledCount: reenabledCount,
65+
QuarantineDaysDebt: quarantineDaysDebt,
66+
GeneratedAt: now,
67+
}
68+
}
69+
70+
// WriteReport writes the report to both GitHub Actions summary and skipper-report.json.
71+
func WriteReport(report *Report) error {
72+
if report == nil {
73+
return fmt.Errorf("report is nil")
74+
}
75+
76+
// Write to skipper-report.json
77+
data, err := json.MarshalIndent(report, "", " ")
78+
if err != nil {
79+
return fmt.Errorf("could not marshal report: %w", err)
80+
}
81+
82+
if err := os.WriteFile("skipper-report.json", data, 0644); err != nil {
83+
Warn(fmt.Sprintf("could not write skipper-report.json: %v", err))
84+
} else {
85+
Logf("wrote skipper-report.json")
86+
}
87+
88+
// Write to GitHub Actions job summary if env var is set
89+
if summaryFile := os.Getenv("GITHUB_STEP_SUMMARY"); summaryFile != "" {
90+
markdown := formatReportMarkdown(report)
91+
if err := os.WriteFile(summaryFile, []byte(markdown), 0644); err != nil {
92+
Warn(fmt.Sprintf("could not write to GITHUB_STEP_SUMMARY: %v", err))
93+
} else {
94+
Logf("wrote quarantine report to GITHUB_STEP_SUMMARY")
95+
}
96+
}
97+
98+
return nil
99+
}
100+
101+
// formatReportMarkdown formats the report as GitHub Actions markdown.
102+
func formatReportMarkdown(report *Report) string {
103+
markdown := "# Quarantine Report\n\n"
104+
105+
markdown += fmt.Sprintf("| Metric | Count |\n|--------|-------|\n")
106+
markdown += fmt.Sprintf("| Disabled Tests | %d |\n", report.DisabledCount)
107+
markdown += fmt.Sprintf("| Reenabled Tests | %d |\n", report.ReenabledCount)
108+
markdown += fmt.Sprintf("| Quarantine Days Debt | %d |\n", report.QuarantineDaysDebt)
109+
110+
if len(report.DueThisWeek) > 0 {
111+
markdown += "\n## Tests Due This Week\n\n"
112+
for _, testID := range report.DueThisWeek {
113+
markdown += fmt.Sprintf("- %s\n", testID)
114+
}
115+
}
116+
117+
markdown += fmt.Sprintf("\n*Generated at: %s*\n", report.GeneratedAt.Format(time.RFC3339))
118+
119+
return markdown
120+
}

‎core/report_test.go‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
package core
2+
3+
import (
4+
"os"
5+
"testing"
6+
"time"
7+
)
8+
9+
func TestGenerateReport(t *testing.T) {
10+
now := time.Now().UTC()
11+
12+
resolver := &SkipperResolver{
13+
cache: map[string]*time.Time{
14+
"test1": ptrTime(now.AddDate(0, 0, 3)), // disabled, due in 3 days
15+
"test2": ptrTime(now.AddDate(0, 0, 10)), // disabled, due in 10 days
16+
"test3": ptrTime(now.AddDate(0, 0, -2)), // reenabled (past date)
17+
"test4": nil, // no date, enabled
18+
},
19+
}
20+
21+
report := GenerateReport(resolver)
22+
23+
if report == nil {
24+
t.Fatalf("GenerateReport returned nil")
25+
}
26+
27+
if report.DisabledCount != 2 {
28+
t.Errorf("expected DisabledCount=2, got %d", report.DisabledCount)
29+
}
30+
31+
if report.ReenabledCount != 1 {
32+
t.Errorf("expected ReenabledCount=1, got %d", report.ReenabledCount)
33+
}
34+
35+
if len(report.DueThisWeek) != 1 {
36+
t.Errorf("expected 1 test due this week, got %d", len(report.DueThisWeek))
37+
}
38+
39+
if report.DueThisWeek[0] != "test1" {
40+
t.Errorf("expected 'test1' in DueThisWeek, got %v", report.DueThisWeek)
41+
}
42+
43+
// Check quarantine days debt: test1 (3 days) + test2 (10 days) = 13
44+
expectedDebt := 3 + 10
45+
if report.QuarantineDaysDebt != expectedDebt {
46+
t.Errorf("expected QuarantineDaysDebt=%d, got %d", expectedDebt, report.QuarantineDaysDebt)
47+
}
48+
}
49+
50+
func TestGenerateReportNilResolver(t *testing.T) {
51+
report := GenerateReport(nil)
52+
53+
if report == nil {
54+
t.Fatalf("GenerateReport(nil) returned nil, expected empty report")
55+
}
56+
57+
if report.DisabledCount != 0 {
58+
t.Errorf("expected DisabledCount=0 for nil resolver, got %d", report.DisabledCount)
59+
}
60+
}
61+
62+
func TestWriteReport(t *testing.T) {
63+
tmpDir := t.TempDir()
64+
originalCwd, err := os.Getwd()
65+
if err != nil {
66+
t.Fatalf("could not get cwd: %v", err)
67+
}
68+
defer os.Chdir(originalCwd)
69+
70+
if err := os.Chdir(tmpDir); err != nil {
71+
t.Fatalf("could not change to temp dir: %v", err)
72+
}
73+
74+
report := &Report{
75+
DisabledCount: 2,
76+
DueThisWeek: []string{"test1", "test2"},
77+
ReenabledCount: 1,
78+
QuarantineDaysDebt: 10,
79+
GeneratedAt: time.Now().UTC(),
80+
}
81+
82+
if err := WriteReport(report); err != nil {
83+
t.Fatalf("WriteReport failed: %v", err)
84+
}
85+
86+
// Check skipper-report.json was created
87+
data, err := os.ReadFile("skipper-report.json")
88+
if err != nil {
89+
t.Fatalf("skipper-report.json not found: %v", err)
90+
}
91+
92+
if len(data) == 0 {
93+
t.Errorf("skipper-report.json is empty")
94+
}
95+
}
96+
97+
func TestWriteReportGitHubActionsIntegration(t *testing.T) {
98+
tmpDir := t.TempDir()
99+
summaryFile := tmpDir + "/summary.txt"
100+
101+
originalCwd, err := os.Getwd()
102+
if err != nil {
103+
t.Fatalf("could not get cwd: %v", err)
104+
}
105+
defer os.Chdir(originalCwd)
106+
107+
if err := os.Chdir(tmpDir); err != nil {
108+
t.Fatalf("could not change to temp dir: %v", err)
109+
}
110+
111+
os.Setenv("GITHUB_STEP_SUMMARY", summaryFile)
112+
defer os.Unsetenv("GITHUB_STEP_SUMMARY")
113+
114+
report := &Report{
115+
DisabledCount: 2,
116+
DueThisWeek: []string{"test1"},
117+
ReenabledCount: 1,
118+
QuarantineDaysDebt: 15,
119+
GeneratedAt: time.Now().UTC(),
120+
}
121+
122+
if err := WriteReport(report); err != nil {
123+
t.Fatalf("WriteReport failed: %v", err)
124+
}
125+
126+
// Check summary file was created
127+
data, err := os.ReadFile(summaryFile)
128+
if err != nil {
129+
t.Fatalf("summary file not found: %v", err)
130+
}
131+
132+
markdown := string(data)
133+
if len(markdown) == 0 {
134+
t.Errorf("summary file is empty")
135+
}
136+
137+
if !contains(markdown, "Quarantine Report") {
138+
t.Errorf("summary missing title")
139+
}
140+
141+
if !contains(markdown, "2") {
142+
t.Errorf("summary missing disabled count")
143+
}
144+
}
145+
146+
func ptrTime(t time.Time) *time.Time {
147+
return &t
148+
}
149+
150+
func contains(s, substr string) bool {
151+
for i := 0; i <= len(s)-len(substr); i++ {
152+
if s[i:i+len(substr)] == substr {
153+
return true
154+
}
155+
}
156+
return false
157+
}

‎ginkgo/skipper.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ import (
2626
ginkgo "github.com/onsi/ginkgo/v2"
2727
)
2828

29+
var (
30+
globalResolverMu sync.Mutex
31+
globalResolver *core.SkipperResolver
32+
)
33+
2934
// RegisterSkipperHooks installs Ginkgo lifecycle hooks that initialize the
3035
// Skipper resolver, skip disabled specs, and (in sync mode) reconcile the
3136
// spreadsheet after the suite finishes.
@@ -53,6 +58,9 @@ func RegisterSkipperHooks(config core.SkipperConfig) {
5358
return nil
5459
}
5560
resolver = r
61+
globalResolverMu.Lock()
62+
globalResolver = r
63+
globalResolverMu.Unlock()
5664
return data
5765
},
5866
func(data []byte) {
@@ -63,6 +71,9 @@ func RegisterSkipperHooks(config core.SkipperConfig) {
6371
return
6472
}
6573
resolver = r
74+
globalResolverMu.Lock()
75+
globalResolver = r
76+
globalResolverMu.Unlock()
6677
},
6778
)
6879

@@ -124,3 +135,17 @@ func RegisterSkipperHooks(config core.SkipperConfig) {
124135
},
125136
)
126137
}
138+
139+
// Report generates and emits the quarantine report.
140+
// Call it after RunSpecs completes in your test function.
141+
func Report() error {
142+
globalResolverMu.Lock()
143+
resolver := globalResolver
144+
globalResolverMu.Unlock()
145+
146+
if resolver == nil {
147+
return nil
148+
}
149+
report := core.GenerateReport(resolver)
150+
return core.WriteReport(report)
151+
}

‎testify/suite.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,3 +142,13 @@ func (s *SkipperSuite) TearDownSuite() {
142142
fmt.Fprintf(os.Stderr, "[skipper] sync failed: %v\n", err)
143143
}
144144
}
145+
146+
// Report generates and emits the quarantine report for this suite.
147+
// Call it from TearDownSuite or after suite.Run in your test function.
148+
func (s *SkipperSuite) Report() error {
149+
if s.resolver == nil {
150+
return nil
151+
}
152+
report := core.GenerateReport(s.resolver)
153+
return core.WriteReport(report)
154+
}

‎testing/skipper.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,11 @@ func (s *SkipperTestMain) finalize(ctx context.Context) error {
152152
}
153153
}
154154

155+
// Generate and emit quarantine report
156+
if err := emitReport(); err != nil {
157+
core.Warn(fmt.Sprintf("could not generate report: %v", err))
158+
}
159+
155160
if globalCacheDir != "" {
156161
core.CacheManager{}.Cleanup(globalCacheDir)
157162
}
@@ -185,3 +190,28 @@ func SkipIfDisabled(t *stdtesting.T) {
185190
t.Skip(msg)
186191
}
187192
}
193+
194+
// Report generates and emits the quarantine report to GitHub Actions job summary
195+
// and skipper-report.json. Call this from TestMain after running tests.
196+
//
197+
// Example:
198+
//
199+
// func TestMain(m *testing.M) {
200+
// s := &skippertest.SkipperTestMain{
201+
// Config: core.SkipperConfig{...},
202+
// }
203+
// code := s.Run(m)
204+
// skippertest.Report() // Generate and emit report
205+
// os.Exit(code)
206+
// }
207+
func Report() error {
208+
return emitReport()
209+
}
210+
211+
func emitReport() error {
212+
if globalResolver == nil {
213+
return nil
214+
}
215+
report := core.GenerateReport(globalResolver)
216+
return core.WriteReport(report)
217+
}

0 commit comments

Comments
 (0)