Skip to content

Commit 3794510

Browse files
edigaryevedigaryev
authored
Separate DataDog event enrichment depending on the payload type (#11)
* Pick up the timestamp value from X-Cirrus-Timestamp * Separate DataDog event enrichment depending on the payload type * CI: run "go test ./..." * Enrich build and task's initializer and audit_event's actor
1 parent b2e7ec7 commit 3794510

14 files changed

+422
-61
lines changed

‎.cirrus.yml

+8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
container:
2+
image: golang:latest
3+
4+
task:
5+
name: Test
6+
test_script:
7+
- go test ./...
8+
19
task:
210
name: Release Binaries
311
only_if: $CIRRUS_TAG != ''

‎go.mod

+4
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,22 @@ require (
1111
github.com/deckarep/golang-set/v2 v2.6.0
1212
github.com/labstack/echo/v4 v4.12.0
1313
github.com/spf13/cobra v1.8.1
14+
github.com/stretchr/testify v1.8.4
1415
go.uber.org/zap v1.27.0
1516
)
1617

1718
require (
1819
github.com/DataDog/zstd v1.5.5 // indirect
1920
github.com/Microsoft/go-winio v0.6.2 // indirect
21+
github.com/davecgh/go-spew v1.1.1 // indirect
2022
github.com/goccy/go-json v0.10.3 // indirect
2123
github.com/google/go-cmp v0.6.0 // indirect
2224
github.com/google/uuid v1.6.0 // indirect
2325
github.com/inconshreveable/mousetrap v1.1.0 // indirect
2426
github.com/labstack/gommon v0.4.2 // indirect
2527
github.com/mattn/go-colorable v0.1.13 // indirect
2628
github.com/mattn/go-isatty v0.0.20 // indirect
29+
github.com/pmezard/go-difflib v1.0.0 // indirect
2730
github.com/spf13/pflag v1.0.5 // indirect
2831
github.com/valyala/bytebufferpool v1.0.0 // indirect
2932
github.com/valyala/fasttemplate v1.2.2 // indirect
@@ -33,4 +36,5 @@ require (
3336
golang.org/x/oauth2 v0.21.0 // indirect
3437
golang.org/x/sys v0.22.0 // indirect
3538
golang.org/x/text v0.16.0 // indirect
39+
gopkg.in/yaml.v3 v3.0.1 // indirect
3640
)

‎go.sum

+1
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
114114
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
115115
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
116116
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
117+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
117118
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
118119
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
119120
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎internal/command/datadog/datadog.go

+18-61
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"errors"
99
"fmt"
1010
"github.com/brpaz/echozap"
11+
payloadpkg "github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload"
1112
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
1213
mapset "github.com/deckarep/golang-set/v2"
1314
"github.com/labstack/echo/v4"
@@ -33,25 +34,6 @@ var (
3334
ErrSignatureVerificationFailed = errors.New("event signature verification failed")
3435
)
3536

36-
type commonWebhookFields struct {
37-
Action *string
38-
Timestamp *int64
39-
Actor struct {
40-
ID *int64
41-
}
42-
Repository struct {
43-
ID *int64
44-
Owner *string
45-
Name *string
46-
}
47-
Build struct {
48-
ID *int64
49-
}
50-
Task struct {
51-
ID *int64
52-
}
53-
}
54-
5537
func NewCommand() *cobra.Command {
5638
cmd := &cobra.Command{
5739
Use: "datadog",
@@ -182,7 +164,23 @@ func processWebhookEvent(
182164
}
183165

184166
// Enrich the event with tags
185-
enrichEventWithTags(body, evt, logger)
167+
var payload payloadpkg.Payload
168+
169+
switch presentedEventType {
170+
case "audit_event":
171+
payload = &payloadpkg.AuditEvent{}
172+
case "build", "task":
173+
payload = &payloadpkg.BuildOrTask{}
174+
}
175+
176+
if payload != nil {
177+
if err = json.Unmarshal(body, payload); err != nil {
178+
logger.Warnf("failed to enrich Datadog event with tags: "+
179+
"failed to parse the webhook event of type %q as JSON: %v", presentedEventType, err)
180+
} else {
181+
payload.Enrich(ctx.Request().Header, evt, logger)
182+
}
183+
}
186184

187185
// Datadog silently discards log events submitted with a
188186
// timestamp that is more than 18 hours in the past, sigh.
@@ -227,44 +225,3 @@ func verifyEvent(ctx echo.Context, body []byte) error {
227225

228226
return nil
229227
}
230-
231-
func enrichEventWithTags(body []byte, evt *datadogsender.Event, logger *zap.SugaredLogger) {
232-
var commonWebhookFields commonWebhookFields
233-
234-
if err := json.Unmarshal(body, &commonWebhookFields); err != nil {
235-
logger.Warnf("failed to enrich Datadog event with tags: "+
236-
"failed to parse the webhook event as JSON: %v", err)
237-
238-
return
239-
}
240-
241-
if value := commonWebhookFields.Action; value != nil {
242-
evt.Tags = append(evt.Tags, fmt.Sprintf("action:%s", *value))
243-
}
244-
245-
if timestamp := commonWebhookFields.Timestamp; timestamp != nil {
246-
evt.Timestamp = time.UnixMilli(*timestamp).UTC()
247-
}
248-
249-
if value := commonWebhookFields.Actor.ID; value != nil {
250-
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_id:%d", *value))
251-
}
252-
253-
if value := commonWebhookFields.Repository.ID; value != nil {
254-
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_id:%d", *value))
255-
}
256-
if value := commonWebhookFields.Repository.Owner; value != nil {
257-
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_owner:%s", *value))
258-
}
259-
if value := commonWebhookFields.Repository.Name; value != nil {
260-
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_name:%s", *value))
261-
}
262-
263-
if value := commonWebhookFields.Build.ID; value != nil {
264-
evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value))
265-
}
266-
267-
if value := commonWebhookFields.Task.ID; value != nil {
268-
evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value))
269-
}
270-
}

‎internal/command/datadog/payload/auditevent.go

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package payload
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
7+
"go.uber.org/zap"
8+
"net/http"
9+
)
10+
11+
type AuditEvent struct {
12+
Data *string `json:"data"`
13+
14+
Actor struct {
15+
Username *string `json:"username"`
16+
} `json:"actor"`
17+
18+
ActorLocationIP *string `json:"actorLocationIp"`
19+
20+
common
21+
}
22+
23+
func (auditEvent AuditEvent) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
24+
auditEvent.common.Enrich(header, evt, logger)
25+
26+
if data := auditEvent.Data; data != nil {
27+
var auditEventData auditEventData
28+
29+
if err := json.Unmarshal([]byte(*data), &auditEventData); err != nil {
30+
logger.Warnf("failed to unmarshal audit event's data: %v", err)
31+
32+
return
33+
} else {
34+
auditEventData.Enrich(header, evt, logger)
35+
}
36+
}
37+
38+
actorUsername := "api"
39+
if value := auditEvent.Actor.Username; value != nil {
40+
actorUsername = *value
41+
}
42+
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_username:%s", actorUsername))
43+
44+
if value := auditEvent.ActorLocationIP; value != nil {
45+
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_location_ip:%s", *value))
46+
}
47+
}

‎internal/command/datadog/payload/auditevent_test.go

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package payload_test
2+
3+
import (
4+
"encoding/json"
5+
"github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload"
6+
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
7+
"github.com/stretchr/testify/require"
8+
"go.uber.org/zap"
9+
"net/http"
10+
"os"
11+
"path/filepath"
12+
"strconv"
13+
"testing"
14+
"time"
15+
)
16+
17+
func TestEnrichAuditEvent(t *testing.T) {
18+
body, err := os.ReadFile(filepath.Join("testdata", "audit_event.json"))
19+
require.NoError(t, err)
20+
21+
evt := &datadogsender.Event{}
22+
23+
payload := payload.AuditEvent{}
24+
require.NoError(t, json.Unmarshal(body, &payload))
25+
payload.Enrich(http.Header{
26+
"X-Cirrus-Timestamp": []string{strconv.FormatInt(time.Now().UnixMilli(), 10)},
27+
}, evt, zap.S())
28+
require.WithinDuration(t, time.Now(), evt.Timestamp, time.Second)
29+
require.Equal(t, []string{
30+
"action:created",
31+
"type:graphql.mutation",
32+
"data.mutationName:GenerateNewScopedAccessToken",
33+
"actor_username:edigaryev",
34+
"actor_location_ip:1.2.3.4",
35+
}, evt.Tags)
36+
}

‎internal/command/datadog/payload/auditeventdata.go

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package payload
2+
3+
import (
4+
"fmt"
5+
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
6+
"go.uber.org/zap"
7+
"net/http"
8+
)
9+
10+
type auditEventData struct {
11+
MutationName *string `json:"mutationName"`
12+
BuildID *string `json:"buildId"`
13+
TaskID *string `json:"taskId"`
14+
}
15+
16+
func (auditEventData auditEventData) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
17+
if mutationName := auditEventData.MutationName; mutationName != nil {
18+
evt.Tags = append(evt.Tags, fmt.Sprintf("data.mutationName:%s", *mutationName))
19+
}
20+
21+
if buildID := auditEventData.BuildID; buildID != nil {
22+
evt.Tags = append(evt.Tags, fmt.Sprintf("data.buildId:%s", *buildID))
23+
}
24+
25+
if taskID := auditEventData.TaskID; taskID != nil {
26+
evt.Tags = append(evt.Tags, fmt.Sprintf("data.taskId:%s", *taskID))
27+
}
28+
}

‎internal/command/datadog/payload/buildortask.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package payload
2+
3+
import (
4+
"fmt"
5+
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
6+
"go.uber.org/zap"
7+
"net/http"
8+
"strings"
9+
)
10+
11+
type BuildOrTask struct {
12+
Build struct {
13+
ID *int64 `json:"id"`
14+
Status *string `json:"status"`
15+
Branch *string `json:"branch"`
16+
PullRequest *int64 `json:"pullRequest"`
17+
User struct {
18+
Username *string `json:"username"`
19+
} `json:"user"`
20+
}
21+
Task struct {
22+
ID *int64 `json:"id"`
23+
Name *string `json:"name"`
24+
Status *string `json:"status"`
25+
InstanceType *string `json:"instanceType"`
26+
UniqueLabels []string `json:"uniqueLabels"`
27+
}
28+
29+
common
30+
}
31+
32+
func (buildOrTask BuildOrTask) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
33+
buildOrTask.common.Enrich(header, evt, logger)
34+
35+
if value := buildOrTask.Build.ID; value != nil {
36+
evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value))
37+
}
38+
if value := buildOrTask.Build.Status; value != nil {
39+
evt.Tags = append(evt.Tags, fmt.Sprintf("build_status:%s", *value))
40+
}
41+
if value := buildOrTask.Build.Branch; value != nil {
42+
evt.Tags = append(evt.Tags, fmt.Sprintf("build_branch:%s", *value))
43+
}
44+
if value := buildOrTask.Build.PullRequest; value != nil {
45+
evt.Tags = append(evt.Tags, fmt.Sprintf("build_pull_request:%d", *value))
46+
}
47+
48+
initializerUsername := "api"
49+
if value := buildOrTask.Build.User.Username; value != nil {
50+
initializerUsername = *value
51+
}
52+
evt.Tags = append(evt.Tags, fmt.Sprintf("initializer_username:%s", initializerUsername))
53+
54+
if value := buildOrTask.Task.ID; value != nil {
55+
evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value))
56+
}
57+
if value := buildOrTask.Task.Name; value != nil {
58+
evt.Tags = append(evt.Tags, fmt.Sprintf("task_name:%s", *value))
59+
}
60+
if value := buildOrTask.Task.Status; value != nil {
61+
evt.Tags = append(evt.Tags, fmt.Sprintf("task_status:%s", *value))
62+
}
63+
if value := buildOrTask.Task.InstanceType; value != nil {
64+
evt.Tags = append(evt.Tags, fmt.Sprintf("task_instance_type:%s", *value))
65+
}
66+
if value := buildOrTask.Task.UniqueLabels; len(value) > 0 {
67+
evt.Tags = append(evt.Tags, fmt.Sprintf("task_unique_labels:%s", strings.Join(value, ",")))
68+
}
69+
}

0 commit comments

Comments
 (0)