Skip to content
This repository was archived by the owner on May 29, 2024. It is now read-only.

Commit 346ec30

Browse files
adrain-cbadrain-cb
authored
Publish SNS event on new alert (#200)
1 parent ff9d7d3 commit 346ec30

24 files changed

+574
-45
lines changed

‎.github/workflows/hygeine.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ jobs:
4848
- name: Set up Node.js
4949
uses: actions/setup-node@v2
5050
with:
51-
node-version: '20'
52-
51+
node-version: '20'
52+
5353
- name: Install markdownlint CLI
5454
run: npm install -g markdownlint-cli
5555

‎Makefile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@ go-gen-mocks:
2727

2828
.PHONY: test
2929
test:
30-
@ go test ./internal/... -timeout $(TEST_LIMIT)
30+
@go test ./internal/... -timeout $(TEST_LIMIT)
3131

3232
.PHONY: test-e2e
3333
e2e-test:
34-
@ go test ./e2e/... -timeout $(TEST_LIMIT) -deploy-config ../.devnet/devnetL1.json -parallel=4 -v
34+
@docker compose up -d
35+
@go test ./e2e/... -timeout $(TEST_LIMIT) -deploy-config ../.devnet/devnetL1.json -parallel=4 -v
36+
@docker compose down
3537

3638
.PHONY: lint
3739
lint:

‎config.env.template

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ P0_PAGERDUTY_ALERT_EVENTS_URL=
3232
P1_PAGERDUTY_INTEGRATION_KEY=
3333
P1_PAGERDUTY_ALERT_EVENTS_URL=
3434

35+
SNS_TOPIC_ARN=
36+
AWS_ENDPOINT=
37+
3538
# Metrics configurations
3639
METRICS_HOST=localhost
3740
METRICS_PORT=7300

‎docker-compose.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
version: "3.8"
2+
3+
services:
4+
localstack:
5+
container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
6+
image: localstack/localstack:3.1.0
7+
ports:
8+
- "127.0.0.1:4566:4566" # LocalStack Gateway
9+
- "127.0.0.1:4510-4559:4510-4559" # external services port range
10+
environment:
11+
# LocalStack configuration: https://docs.localstack.cloud/references/configuration/
12+
- DEBUG=${DEBUG:-0}
13+
volumes:
14+
- "/var/run/docker.sock:/var/run/docker.sock"
15+
- "./scripts/localstack-e2e-test-setup.sh:/etc/localstack/init/ready.d/script.sh"

‎docs/alert-routing.md

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@ a few examples on how you might want to configure your alert routing.
3232

3333
Pessimism currently supports the following alert destinations:
3434

35-
| Name | Description |
36-
|-----------|-------------------------------------|
37-
| slack | Sends alerts to a Slack channel |
38-
| pagerduty | Sends alerts to a PagerDuty service |
35+
| Name | Description |
36+
|-----------|---------------------------------------------------|
37+
| slack | Sends alerts to a Slack channel |
38+
| pagerduty | Sends alerts to a PagerDuty service |
39+
| sns | Sends alerts to an SNS topic defined in .env file |
3940

4041
## Alert Severity
4142

@@ -47,6 +48,16 @@ Pessimism currently defines the following severities for alerts:
4748
| medium | Alerts that could be hazardous, but may not be completely destructive |
4849
| high | Alerts that require immediate attention and could result in a loss of funds |
4950

51+
## Publishing to an SNS Topic
52+
53+
To publish alerts to an SNS topic, you must first create an SNS topic in the AWS
54+
console. Once you have created the topic, you will need to add the ARN of the
55+
topic to the `.env` file. Ensure that you have AWS_REGION,
56+
`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` set in your environment if you are looking to publish messages to an SNS
57+
topic. The ARN should be added to the `SNS_TOPIC_ARN` variable found in the `.env` file.
58+
The AWS_ENDPOINT is optional and is primarily used for testing with localstack.
59+
> Note: Currently, Pessimism only support one SNS topic to publish alerts to.
60+
5061
## PagerDuty Severity Mapping
5162

5263
PagerDuty supports the following severities: `critical`, `error`, `warning`,

‎e2e/alerting_test.go

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package e2e_test
33
import (
44
"context"
55
"math/big"
6+
67
"testing"
78
"time"
89

@@ -17,11 +18,18 @@ import (
1718
"github.com/stretchr/testify/require"
1819
)
1920

21+
const (
22+
// These are localstack specific Topic ARNs that are used to test the SNS integration.
23+
MultiDirectiveTopicArn = "arn:aws:sns:us-east-1:000000000000:multi-directive-test-topic"
24+
CoolDownTopicArn = "arn:aws:sns:us-east-1:000000000000:alert-cooldown-test-topic"
25+
)
26+
2027
// TestMultiDirectiveRouting ... Tests the E2E flow of a contract event heuristic with high priority alerts all
2128
// necessary destinations
2229
func TestMultiDirectiveRouting(t *testing.T) {
2330

24-
ts := e2e.CreateSysTestSuite(t)
31+
ts := e2e.CreateSysTestSuite(t, MultiDirectiveTopicArn)
32+
defer ts.Close()
2533

2634
updateSig := "ConfigUpdate(uint256,uint8,bytes)"
2735
alertMsg := "System config gas config updated"
@@ -73,6 +81,12 @@ func TestMultiDirectiveRouting(t *testing.T) {
7381
return height != nil && height.Uint64() > receipt.BlockNumber.Uint64(), nil
7482
}))
7583

84+
snsMessages, err := e2e.GetSNSMessages(ts.AppCfg.AlertConfig.SNSConfig.Endpoint, "multi-directive-test-queue")
85+
require.NoError(t, err)
86+
87+
assert.Len(t, snsMessages.Messages, 1, "Incorrect number of SNS messages sent")
88+
assert.Contains(t, *snsMessages.Messages[0].Body, "contract_event", "System contract event alert was not sent")
89+
7690
slackPosts := ts.TestSlackSvr.SlackAlerts()
7791
pdPosts := ts.TestPagerDutyServer.PagerDutyAlerts()
7892

@@ -90,7 +104,7 @@ func TestMultiDirectiveRouting(t *testing.T) {
90104
// balance enforcement heuristic session on L2 network with a cooldown.
91105
func TestCoolDown(t *testing.T) {
92106

93-
ts := e2e.CreateSysTestSuite(t)
107+
ts := e2e.CreateSysTestSuite(t, CoolDownTopicArn)
94108
defer ts.Close()
95109

96110
alice := ts.Cfg.Secrets.Addresses().Alice
@@ -149,7 +163,7 @@ func TestCoolDown(t *testing.T) {
149163
receipt, err := wait.ForReceipt(context.Background(), ts.L2Client, drainAliceTx.Hash(), types.ReceiptStatusSuccessful)
150164
require.NoError(t, err)
151165

152-
require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
166+
require.NoError(t, wait.For(context.Background(), 1000*time.Millisecond, func() (bool, error) {
153167
id := ids[0].PathID
154168
height, err := ts.Subsystems.PathHeight(id)
155169
if err != nil {
@@ -162,6 +176,11 @@ func TestCoolDown(t *testing.T) {
162176
// Check that the balance enforcement was triggered using the mocked server cache.
163177
posts := ts.TestSlackSvr.SlackAlerts()
164178

179+
sqsMessages, err := e2e.GetSNSMessages(ts.AppCfg.AlertConfig.SNSConfig.Endpoint, "alert-cooldown-test-queue")
180+
assert.NoError(t, err)
181+
assert.Len(t, sqsMessages.Messages, 1, "Incorrect number of SNS messages sent")
182+
assert.Contains(t, *sqsMessages.Messages[0].Body, "balance_enforcement", "Balance enforcement alert was not sent")
183+
165184
require.Equal(t, 1, len(posts), "No balance enforcement alert was sent")
166185
assert.Contains(t, posts[0].Text, "balance_enforcement", "Balance enforcement alert was not sent")
167186
assert.Contains(t, posts[0].Text, alertMsg)
@@ -170,4 +189,5 @@ func TestCoolDown(t *testing.T) {
170189
time.Sleep(1 * time.Second)
171190
posts = ts.TestSlackSvr.SlackAlerts()
172191
assert.Equal(t, 1, len(posts), "No alerts should be sent after the transaction is sent")
192+
173193
}

‎e2e/heuristic_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import (
3131
// balance enforcement heuristic session on L2 network.
3232
func TestBalanceEnforcement(t *testing.T) {
3333

34-
ts := e2e.CreateSysTestSuite(t)
34+
ts := e2e.CreateSysTestSuite(t, "")
3535
defer ts.Close()
3636

3737
alice := ts.Cfg.Secrets.Addresses().Alice
@@ -158,7 +158,7 @@ func TestBalanceEnforcement(t *testing.T) {
158158
// contract event heuristic session on L1 network.
159159
func TestContractEvent(t *testing.T) {
160160

161-
ts := e2e.CreateSysTestSuite(t)
161+
ts := e2e.CreateSysTestSuite(t, "")
162162
defer ts.Close()
163163

164164
// The string declaration of the event we want to listen for.
@@ -226,7 +226,7 @@ func TestContractEvent(t *testing.T) {
226226
// safety heuristic session. This test ensures that an alert is produced in the event
227227
// of a highly suspicious withdrawal.
228228
func TestWithdrawalSafetyAllInvariants(t *testing.T) {
229-
ts := e2e.CreateSysTestSuite(t)
229+
ts := e2e.CreateSysTestSuite(t, "")
230230
defer ts.Close()
231231

232232
opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.Alice, ts.Cfg.L2ChainIDBig())
@@ -357,7 +357,7 @@ func TestWithdrawalSafetyAllInvariants(t *testing.T) {
357357
// of a normal tx
358358
func TestWithdrawalSafetyNoInvariants(t *testing.T) {
359359

360-
ts := e2e.CreateSysTestSuite(t)
360+
ts := e2e.CreateSysTestSuite(t, "")
361361
defer ts.Close()
362362

363363
ids, err := ts.App.BootStrap([]*models.SessionRequestParams{
@@ -439,7 +439,7 @@ func TestWithdrawalSafetyNoInvariants(t *testing.T) {
439439
// TestFaultDetector ... Ensures that an alert is produced in the presence of a faulty L2Output root
440440
// on the L1 Optimism portal contract.
441441
func TestFaultDetector(t *testing.T) {
442-
ts := e2e.CreateSysTestSuite(t)
442+
ts := e2e.CreateSysTestSuite(t, "")
443443
defer ts.Close()
444444

445445
// Generate transactor opts

‎e2e/setup.go

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import (
66
"os"
77
"testing"
88

9+
"github.com/aws/aws-sdk-go/aws"
10+
"github.com/aws/aws-sdk-go/aws/session"
11+
"github.com/aws/aws-sdk-go/service/sqs"
912
"github.com/base-org/pessimism/internal/alert"
1013
"github.com/base-org/pessimism/internal/api/server"
1114
"github.com/base-org/pessimism/internal/app"
@@ -19,11 +22,11 @@ import (
1922
"github.com/base-org/pessimism/internal/state"
2023
"github.com/base-org/pessimism/internal/subsystem"
2124
ix_node "github.com/ethereum-optimism/optimism/indexer/node"
22-
"github.com/golang/mock/gomock"
23-
2425
op_e2e "github.com/ethereum-optimism/optimism/op-e2e"
2526
"github.com/ethereum/go-ethereum/ethclient"
2627
"github.com/ethereum/go-ethereum/log"
28+
"github.com/golang/mock/gomock"
29+
"go.uber.org/zap"
2730
)
2831

2932
// SysTestSuite ... Stores all the information needed to run an e2e system test
@@ -51,7 +54,7 @@ type SysTestSuite struct {
5154
}
5255

5356
// CreateSysTestSuite ... Creates a new SysTestSuite
54-
func CreateSysTestSuite(t *testing.T) *SysTestSuite {
57+
func CreateSysTestSuite(t *testing.T, topicArn string) *SysTestSuite {
5558
t.Log("Creating system test suite")
5659
ctx := context.Background()
5760
logging.New(core.Development)
@@ -112,11 +115,14 @@ func CreateSysTestSuite(t *testing.T) *SysTestSuite {
112115

113116
pagerdutyServer := NewTestPagerDutyServer("127.0.0.1", 0)
114117

118+
setAwsVars(t)
119+
115120
slackURL := fmt.Sprintf("http://127.0.0.1:%d", slackServer.Port)
116121
pagerdutyURL := fmt.Sprintf("http://127.0.0.1:%d", pagerdutyServer.Port)
117122

118123
appCfg.AlertConfig.PagerdutyAlertEventsURL = pagerdutyURL
119124
appCfg.AlertConfig.RoutingParams = DefaultRoutingParams(core.StringFromEnv(slackURL))
125+
appCfg.AlertConfig.SNSConfig.TopicArn = topicArn
120126

121127
pess, kill, err := app.NewPessimismApp(ctx, appCfg)
122128
if err != nil {
@@ -165,6 +171,9 @@ func DefaultTestConfig() *config.Config {
165171
AlertConfig: &alert.Config{
166172
PagerdutyAlertEventsURL: "",
167173
RoutingCfgPath: "",
174+
SNSConfig: &client.SNSConfig{
175+
Endpoint: "http://localhost:4566",
176+
},
168177
},
169178
EngineConfig: &engine.Config{
170179
WorkerCount: workerCount,
@@ -186,6 +195,52 @@ func DefaultTestConfig() *config.Config {
186195
}
187196
}
188197

198+
func setAwsVars(t *testing.T) {
199+
awsEnvVariables := map[string]string{
200+
"AWS_REGION": "us-east-1",
201+
"AWS_SECRET_ACCESS_KEY": "test",
202+
"AWS_ACCESS_KEY_ID": "test",
203+
}
204+
for key, value := range awsEnvVariables {
205+
if err := os.Setenv(key, value); err != nil {
206+
t.Fatalf("Error setting %s environment variable: %s", key, err)
207+
}
208+
}
209+
}
210+
211+
func GetSNSMessages(endpoint string, queueName string) (*sqs.ReceiveMessageOutput, error) {
212+
sess, err := session.NewSession(&aws.Config{
213+
Endpoint: aws.String(endpoint),
214+
})
215+
if err != nil {
216+
logging.NoContext().Error("failed to create AWS session", zap.Error(err))
217+
return nil, err
218+
}
219+
220+
svc := sqs.New(sess)
221+
urlResult, err := svc.GetQueueUrl(&sqs.GetQueueUrlInput{
222+
QueueName: aws.String(queueName),
223+
})
224+
if err != nil {
225+
return nil, err
226+
}
227+
228+
queueURL := urlResult.QueueUrl
229+
msgResult, err := svc.ReceiveMessage(&sqs.ReceiveMessageInput{
230+
QueueUrl: queueURL,
231+
MaxNumberOfMessages: aws.Int64(10),
232+
WaitTimeSeconds: aws.Int64(5),
233+
MessageAttributeNames: []*string{
234+
aws.String(sqs.QueueAttributeNameAll),
235+
},
236+
})
237+
if err != nil {
238+
return nil, err
239+
}
240+
241+
return msgResult, nil
242+
}
243+
189244
// DefaultRoutingParams ... Returns a default routing params configuration for testing
190245
func DefaultRoutingParams(slackURL core.StringFromEnv) *core.AlertRoutingParams {
191246
return &core.AlertRoutingParams{

‎go.mod

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ require (
2525
github.com/Microsoft/go-winio v0.6.1 // indirect
2626
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
2727
github.com/ajg/form v1.5.1 // indirect
28+
github.com/aws/aws-sdk-go v1.50.3 // indirect
2829
github.com/benbjohnson/clock v1.3.5 // indirect
2930
github.com/beorn7/perks v1.0.1 // indirect
3031
github.com/bits-and-blooms/bitset v1.7.0 // indirect
@@ -50,13 +51,17 @@ require (
5051
github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
5152
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
5253
github.com/deepmap/oapi-codegen v1.8.2 // indirect
54+
github.com/distribution/reference v0.5.0 // indirect
5355
github.com/dlclark/regexp2 v1.7.0 // indirect
56+
github.com/docker/docker v25.0.2+incompatible // indirect
57+
github.com/docker/go-connections v0.5.0 // indirect
5458
github.com/docker/go-units v0.5.0 // indirect
5559
github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect
5660
github.com/elastic/gosigar v0.14.2 // indirect
5761
github.com/ethereum-optimism/go-ethereum-hdwallet v0.1.3 // indirect
5862
github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20231001123245-7b48d3818686 // indirect
5963
github.com/ethereum/c-kzg-4844 v0.3.1 // indirect
64+
github.com/felixge/httpsnoop v1.0.4 // indirect
6065
github.com/fjl/memsize v0.0.1 // indirect
6166
github.com/flynn/noise v1.0.0 // indirect
6267
github.com/francoispqt/gojay v1.2.13 // indirect
@@ -65,6 +70,8 @@ require (
6570
github.com/getsentry/sentry-go v0.18.0 // indirect
6671
github.com/go-chi/chi/v5 v5.0.10 // indirect
6772
github.com/go-chi/docgen v1.2.0 // indirect
73+
github.com/go-logr/logr v1.4.1 // indirect
74+
github.com/go-logr/stdr v1.2.2 // indirect
6875
github.com/go-ole/go-ole v1.2.6 // indirect
6976
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
7077
github.com/go-stack/stack v1.8.1 // indirect
@@ -74,7 +81,7 @@ require (
7481
github.com/gogo/protobuf v1.3.2 // indirect
7582
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
7683
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
77-
github.com/google/go-cmp v0.5.9 // indirect
84+
github.com/google/go-cmp v0.6.0 // indirect
7885
github.com/google/gopacket v1.1.19 // indirect
7986
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
8087
github.com/gorilla/websocket v1.5.0 // indirect
@@ -104,6 +111,7 @@ require (
104111
github.com/jbenet/goprocess v0.1.4 // indirect
105112
github.com/jinzhu/inflection v1.0.0 // indirect
106113
github.com/jinzhu/now v1.1.5 // indirect
114+
github.com/jmespath/go-jmespath v0.4.0 // indirect
107115
github.com/klauspost/compress v1.16.7 // indirect
108116
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
109117
github.com/koron/go-ssdp v0.0.4 // indirect
@@ -148,6 +156,8 @@ require (
148156
github.com/multiformats/go-varint v0.0.7 // indirect
149157
github.com/onsi/ginkgo/v2 v2.12.0 // indirect
150158
github.com/onsi/gomega v1.28.0 // indirect
159+
github.com/opencontainers/go-digest v1.0.0 // indirect
160+
github.com/opencontainers/image-spec v1.0.2 // indirect
151161
github.com/opencontainers/runtime-spec v1.1.0 // indirect
152162
github.com/opentracing/opentracing-go v1.2.0 // indirect
153163
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
@@ -178,6 +188,10 @@ require (
178188
github.com/urfave/cli/v2 v2.25.7 // indirect
179189
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
180190
github.com/yusufpapurcu/wmi v1.2.2 // indirect
191+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
192+
go.opentelemetry.io/otel v1.22.0 // indirect
193+
go.opentelemetry.io/otel/metric v1.22.0 // indirect
194+
go.opentelemetry.io/otel/trace v1.22.0 // indirect
181195
go.uber.org/dig v1.17.0 // indirect
182196
go.uber.org/fx v1.20.0 // indirect
183197
go.uber.org/multierr v1.11.0 // indirect

0 commit comments

Comments
 (0)