From 8b58d7eec3c7b3468ce8fd9963832b9391a1ab38 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Thu, 25 Jan 2024 16:51:10 -0600 Subject: [PATCH 01/10] Initial implementation of blob archiver service Co-authored-by: Qi Wu --- .env.template | 22 ++ .gitignore | 6 + Dockerfile | 18 ++ Makefile | 35 +++ README.md | 52 ++++ api/Makefile | 13 + api/cmd/main.go | 73 +++++ api/flags/config.go | 45 +++ api/flags/flags.go | 34 +++ api/metrics/metrics.go | 39 +++ api/service/api.go | 239 ++++++++++++++++ api/service/api_test.go | 253 +++++++++++++++++ api/service/service.go | 89 ++++++ archiver/Makefile | 13 + archiver/cmd/main.go | 74 +++++ archiver/flags/config.go | 53 ++++ archiver/flags/flags.go | 40 +++ archiver/metrics/metrics.go | 49 ++++ archiver/service/archiver.go | 223 +++++++++++++++ archiver/service/archiver_test.go | 252 +++++++++++++++++ common/beacon/beacontest/stub.go | 86 ++++++ common/beacon/client.go | 26 ++ common/blobtest/helpers.go | 47 ++++ common/flags/config.go | 122 +++++++++ common/flags/flags.go | 80 ++++++ common/storage/file.go | 72 +++++ common/storage/file_test.go | 115 ++++++++ common/storage/s3.go | 100 +++++++ common/storage/s3_test.go | 55 ++++ common/storage/storage.go | 78 ++++++ common/storage/storage_test.go | 44 +++ common/storage/storagetest/stub.go | 46 ++++ docker-compose.yml | 46 ++++ go.mod | 109 ++++++++ go.sum | 421 +++++++++++++++++++++++++++++ 35 files changed, 3069 insertions(+) create mode 100644 .env.template create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 api/Makefile create mode 100644 api/cmd/main.go create mode 100644 api/flags/config.go create mode 100644 api/flags/flags.go create mode 100644 api/metrics/metrics.go create mode 100644 api/service/api.go create mode 100644 api/service/api_test.go create mode 100644 api/service/service.go create mode 100644 archiver/Makefile create mode 100644 archiver/cmd/main.go create mode 100644 archiver/flags/config.go create mode 100644 archiver/flags/flags.go create mode 100644 archiver/metrics/metrics.go create mode 100644 archiver/service/archiver.go create mode 100644 archiver/service/archiver_test.go create mode 100644 common/beacon/beacontest/stub.go create mode 100644 common/beacon/client.go create mode 100644 common/blobtest/helpers.go create mode 100644 common/flags/config.go create mode 100644 common/flags/flags.go create mode 100644 common/storage/file.go create mode 100644 common/storage/file_test.go create mode 100644 common/storage/s3.go create mode 100644 common/storage/s3_test.go create mode 100644 common/storage/storage.go create mode 100644 common/storage/storage_test.go create mode 100644 common/storage/storagetest/stub.go create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..d76fa06 --- /dev/null +++ b/.env.template @@ -0,0 +1,22 @@ +# To get started, copy this file to .env and set your beacon http endpoint + +BLOB_ARCHIVER_L1_BEACON_HTTP= +BLOB_ARCHIVER_DATA_STORE=s3 +BLOB_ARCHIVER_S3_ENDPOINT=172.17.0.1:9000 +BLOB_ARCHIVER_S3_ACCESS_KEY=admin +BLOB_ARCHIVER_S3_SECRET_ACCESS_KEY=password +BLOB_ARCHIVER_S3_ENDPOINT_HTTPS=false +BLOB_ARCHIVER_S3_BUCKET=blobs +BLOB_ARCHIVER_METRICS_ENABLED=true +BLOB_ARCHIVER_METRICS_PORT=7300 +BLOB_ARCHIVER_ORIGIN_BLOCK=0x0 + +BLOB_API_L1_BEACON_HTTP= +BLOB_API_DATA_STORE=s3 +BLOB_API_S3_ENDPOINT=172.17.0.1:9000 +BLOB_API_S3_ACCESS_KEY=admin +BLOB_API_S3_SECRET_ACCESS_KEY=password +BLOB_API_S3_ENDPOINT_HTTPS=false +BLOB_API_S3_BUCKET=blobs +BLOB_API_METRICS_ENABLED=true +BLOB_API_METRICS_PORT=7301 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c81fc6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.idea/ +.DS_Store +.swp +.env +api/bin +archiver/bin diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3227b02 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.21.6-alpine3.19 as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers jq bash + +WORKDIR /app + +COPY ./go.mod ./go.sum /app/ + +RUN go mod download + +COPY . /app + +RUN make build + +FROM alpine:3.19 + +COPY --from=builder /app/archiver/bin/blob-archiver /usr/local/bin/blob-archiver +COPY --from=builder /app/api/bin/blob-api /usr/local/bin/blob-api \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..605a46b --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +build: + make -C ./archiver blob-archiver + make -C ./api blob-api +.PHONY: build + +build-docker: + docker-compose build +.PHONY: build-docker + +clean: + make -C ./archiver clean + make -C ./api clean +.PHONY: clean + +test: + make -C ./archiver test + make -C ./api test +.PHONY: test + +integration: + docker-compose down + docker-compose up -d minio create-buckets + RUN_INTEGRATION_TESTS=true go test -v ./... +.PHONY: integration + +fmt: + gofmt -s -w . +.PHONY: fmt + +check: fmt clean build build-docker lint test integration +.PHONY: check + +lint: + golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./... +.PHONY: lint \ No newline at end of file diff --git a/README.md b/README.md index 34ab87d..8c65a36 100644 --- a/README.md +++ b/README.md @@ -1 +1,53 @@ # Blob Archiver +The Blob Archiver is a service to archive and query all historical blobs from the beacon chain. It consistens of two +components: + +* **Archiver** - Tracks the beacon chain and writes blobs to a storage backend +* **API** - Implements the blob sidecars [API](https://ethereum.github.io/beacon-APIs/#/Beacon/getBlobSidecars), which +allows clients to retrieve blobs from the storage backend + +### Storage +There are currently two supported storage options: + +* On-disk storage - Blobs are written to disk in a directory +* S3 storage - Blobs are written to an S3 bucket + +You can control which storage backend is used by setting the `BLOB_API_DATA_STORE` and `BLOB_ARCHIVER_DATA_STORE` to +either `disk` or `s3`. + +### Development +The `Makefile` contains a number of commands for development: + +```sh +# Run the tests +make test +# Run the integration tests (will start a local S3 bucket) +make integration + +# Lint the project +make lint + +# Build the project +make build + +# Check all tests, formatting, building +make check +``` + +#### Run Locally +To run the project locally, you should first copy `.env.template` to `.env` and then modify the environment variables +to your beacon client and storage backend of choice. Then you can run the project with: + +```sh +docker-compose up +``` + +You can see a full list of configuration options by running: +```sh +# API +go run api/cmd/main.go + +# Archiver +go run archiver/cmd/main.go + +``` \ No newline at end of file diff --git a/api/Makefile b/api/Makefile new file mode 100644 index 0000000..c4b1e87 --- /dev/null +++ b/api/Makefile @@ -0,0 +1,13 @@ +blob-api: + env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-api ./cmd/main.go + +clean: + rm -f bin/blob-api + +test: + go test -v -race ./... + +.PHONY: \ + blob-api \ + clean \ + test \ No newline at end of file diff --git a/api/cmd/main.go b/api/cmd/main.go new file mode 100644 index 0000000..e736050 --- /dev/null +++ b/api/cmd/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/base-org/blob-archiver/api/flags" + "github.com/base-org/blob-archiver/api/metrics" + "github.com/base-org/blob-archiver/api/service" + "github.com/base-org/blob-archiver/common/beacon" + "github.com/base-org/blob-archiver/common/storage" + opservice "github.com/ethereum-optimism/optimism/op-service" + "github.com/ethereum-optimism/optimism/op-service/cliapp" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + Version = "v0.0.1" + GitCommit = "" + GitDate = "" +) + +func main() { + oplog.SetupDefaults() + + app := cli.NewApp() + app.Flags = cliapp.ProtectFlags(flags.Flags) + app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "") + app.Name = "blob-api" + app.Usage = "API service for Ethereum blobs" + app.Description = "Service for fetching blob sidecars from a datastore" + app.Action = cliapp.LifecycleCmd(Main()) + + err := app.Run(os.Args) + if err != nil { + log.Crit("Application failed", "message", err) + } +} + +// Main is the entrypoint into the API. +// This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed API Server. +func Main() cliapp.LifecycleAction { + return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) { + cfg := flags.ReadConfig(cliCtx) + if err := cfg.Check(); err != nil { + return nil, fmt.Errorf("config check failed: %w", err) + } + + l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig) + oplog.SetGlobalLogHandler(l.GetHandler()) + opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l) + + registry := opmetrics.NewRegistry() + m := metrics.NewMetrics(registry) + + storageClient, err := storage.NewStorage(cfg.StorageConfig, l) + if err != nil { + return nil, err + } + + beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig) + if err != nil { + return nil, err + } + + l.Info("Initializing API Service") + return service.NewAPIService(l, storageClient, beaconClient, cfg, registry, m), nil + } +} diff --git a/api/flags/config.go b/api/flags/config.go new file mode 100644 index 0000000..1d615cc --- /dev/null +++ b/api/flags/config.go @@ -0,0 +1,45 @@ +package flags + +import ( + "fmt" + + common "github.com/base-org/blob-archiver/common/flags" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/urfave/cli/v2" +) + +type APIConfig struct { + LogConfig oplog.CLIConfig + MetricsConfig opmetrics.CLIConfig + BeaconConfig common.BeaconConfig + StorageConfig common.StorageConfig + + ListenAddr string +} + +func (c APIConfig) Check() error { + if err := c.StorageConfig.Check(); err != nil { + return err + } + + if err := c.BeaconConfig.Check(); err != nil { + return err + } + + if c.ListenAddr == "" { + return fmt.Errorf("listen address must be set") + } + + return nil +} + +func ReadConfig(cliCtx *cli.Context) APIConfig { + return APIConfig{ + LogConfig: oplog.ReadCLIConfig(cliCtx), + MetricsConfig: opmetrics.ReadCLIConfig(cliCtx), + BeaconConfig: common.NewBeaconConfig(cliCtx), + StorageConfig: common.NewStorageConfig(cliCtx), + ListenAddr: cliCtx.String(ListenAddressFlag.Name), + } +} diff --git a/api/flags/flags.go b/api/flags/flags.go new file mode 100644 index 0000000..dd74bdc --- /dev/null +++ b/api/flags/flags.go @@ -0,0 +1,34 @@ +package flags + +import ( + common "github.com/base-org/blob-archiver/common/flags" + opservice "github.com/ethereum-optimism/optimism/op-service" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/urfave/cli/v2" +) + +const EnvVarPrefix = "BLOB_API" + +var ( + ListenAddressFlag = &cli.StringFlag{ + Name: "api-list-address", + Usage: "The address to list for new requests on", + EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"), + Value: "0.0.0.0:8000", + } +) + +func init() { + var flags []cli.Flag + + flags = append(flags, common.CLIFlags(EnvVarPrefix)...) + flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...) + flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...) + flags = append(flags, ListenAddressFlag) + + Flags = flags +} + +// Flags contains the list of configuration options available to the binary. +var Flags []cli.Flag diff --git a/api/metrics/metrics.go b/api/metrics/metrics.go new file mode 100644 index 0000000..3a45da0 --- /dev/null +++ b/api/metrics/metrics.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlockIdType string + +var ( + MetricsNamespace = "blob_api" + + BlockIdTypeHash BlockIdType = "hash" + BlockIdTypeBeacon BlockIdType = "beacon" + BlockIdTypeInvalid BlockIdType = "invalid" +) + +type Metricer interface { + RecordBlockIdType(t BlockIdType) +} + +type metricsRecorder struct { + inputType *prometheus.CounterVec +} + +func NewMetrics(registry *prometheus.Registry) Metricer { + factory := metrics.With(registry) + return &metricsRecorder{ + inputType: factory.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "block_id_type", + Help: "The type of block id used to request a block", + }, []string{"type"}), + } +} + +func (m *metricsRecorder) RecordBlockIdType(t BlockIdType) { + m.inputType.WithLabelValues(string(t)).Inc() +} diff --git a/api/service/api.go b/api/service/api.go new file mode 100644 index 0000000..d6c707e --- /dev/null +++ b/api/service/api.go @@ -0,0 +1,239 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "slices" + "strconv" + "strings" + "time" + + client "github.com/attestantio/go-eth2-client" + "github.com/attestantio/go-eth2-client/api" + "github.com/attestantio/go-eth2-client/spec/deneb" + m "github.com/base-org/blob-archiver/api/metrics" + "github.com/base-org/blob-archiver/common/storage" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/log" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" +) + +type httpError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func (e httpError) write(w http.ResponseWriter) { + w.WriteHeader(e.Code) + _ = json.NewEncoder(w).Encode(e) +} + +func (e httpError) Error() string { + return e.Message +} + +const ( + sszAcceptType = "application/octet-stream" + serverTimeout = 60 * time.Second +) + +var ( + errUnknownBlock = &httpError{ + Code: http.StatusNotFound, + Message: "Block not found", + } + errServerError = &httpError{ + Code: http.StatusInternalServerError, + Message: "Internal server error", + } +) + +func newBlockIdError(input string) *httpError { + return &httpError{ + Code: http.StatusBadRequest, + Message: fmt.Sprintf("invalid block id: %s", input), + } +} + +func newIndicesError(input string) *httpError { + return &httpError{ + Code: http.StatusBadRequest, + Message: fmt.Sprintf("invalid index input: %s", input), + } +} + +type API struct { + dataStoreClient storage.DataStoreReader + beaconClient client.BeaconBlockHeadersProvider + router *chi.Mux + logger log.Logger + metrics m.Metricer +} + +func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, metrics m.Metricer, registry *prometheus.Registry, logger log.Logger) *API { + result := &API{ + dataStoreClient: dataStoreClient, + beaconClient: beaconClient, + router: chi.NewRouter(), + logger: logger, + metrics: metrics, + } + + r := result.router + r.Use(middleware.Logger) + r.Use(middleware.Timeout(serverTimeout)) + r.Use(middleware.Recoverer) + r.Use(middleware.Heartbeat("/healthz")) + + recorder := opmetrics.NewPromHTTPRecorder(registry, m.MetricsNamespace) + r.Use(func(handler http.Handler) http.Handler { + return opmetrics.NewHTTPRecordingMiddleware(recorder, handler) + }) + + r.Get("/eth/v1/beacon/blob_sidecars/{id}", result.blobSidecarHandler) + + return result +} + +func isHash(s string) bool { + if len(s) != 66 || !strings.HasPrefix(s, "0x") { + return false + } + + _, err := hexutil.Decode(s) + return err == nil +} + +func isSlot(id string) bool { + _, err := strconv.ParseUint(id, 10, 64) + return err == nil +} + +func isKnownIdentifier(id string) bool { + return slices.Contains([]string{"genesis", "finalized", "head"}, id) +} + +// toBeaconBlockHash converts a string that can be a slot, hash or identifier to a beacon block hash. +func (a *API) toBeaconBlockHash(id string) (common.Hash, *httpError) { + if isHash(id) { + a.metrics.RecordBlockIdType(m.BlockIdTypeHash) + return common.HexToHash(id), nil + } else if isSlot(id) || isKnownIdentifier(id) { + a.metrics.RecordBlockIdType(m.BlockIdTypeBeacon) + result, err := a.beaconClient.BeaconBlockHeader(context.Background(), &api.BeaconBlockHeaderOpts{ + Common: api.CommonOpts{}, + Block: id, + }) + + if err != nil { + var apiErr *api.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 404: + return common.Hash{}, errUnknownBlock + } + } + + return common.Hash{}, errServerError + } + + return common.Hash(result.Data.Root), nil + } else { + a.metrics.RecordBlockIdType(m.BlockIdTypeInvalid) + return common.Hash{}, newBlockIdError(id) + } +} + +func (a *API) blobSidecarHandler(w http.ResponseWriter, r *http.Request) { + param := chi.URLParam(r, "id") + beaconBlockHash, err := a.toBeaconBlockHash(param) + if err != nil { + err.write(w) + return + } + + result, storageErr := a.dataStoreClient.Read(r.Context(), beaconBlockHash) + if storageErr != nil { + if errors.Is(storageErr, storage.ErrNotFound) { + errUnknownBlock.write(w) + } else { + a.logger.Info("unexpected error fetching blobs", "err", storageErr, "beaconBlockHash", beaconBlockHash.String(), "param", param) + errServerError.write(w) + } + return + } + + blobSidecars := result.BlobSidecars + + filteredBlobSidecars, err := filterBlobs(blobSidecars.Data, r.URL.Query().Get("indices")) + if err != nil { + err.write(w) + return + } + + blobSidecars.Data = filteredBlobSidecars + responseType := r.Header.Get("Accept") + + if responseType == sszAcceptType { + res, err := blobSidecars.MarshalSSZ() + if err != nil { + a.logger.Error("unable to marshal blob sidecars to SSZ", "err", err) + errServerError.write(w) + return + } + + _, err = w.Write(res) + + if err != nil { + a.logger.Error("unable to write ssz response", "err", err) + errServerError.write(w) + return + } + } else { + err := json.NewEncoder(w).Encode(blobSidecars) + if err != nil { + a.logger.Error("unable to encode blob sidecars to JSON", "err", err) + errServerError.write(w) + return + } + } +} + +// filterBlobs filters the blobs based on the indices query provided. +// If no indices or invalid indices are provided, the original blobs are returned. +func filterBlobs(blobs []*deneb.BlobSidecar, indices string) ([]*deneb.BlobSidecar, *httpError) { + if indices == "" { + return blobs, nil + } + + splits := strings.Split(indices, ",") + if len(splits) == 0 { + return blobs, nil + } + + indicesMap := map[deneb.BlobIndex]bool{} + for _, index := range splits { + parsedInt, err := strconv.ParseUint(index, 10, 64) + if err != nil { + return nil, newIndicesError(index) + } + blobIndex := deneb.BlobIndex(parsedInt) + indicesMap[blobIndex] = true + } + + filteredBlobs := make([]*deneb.BlobSidecar, 0) + for _, blob := range blobs { + if _, ok := indicesMap[blob.Index]; ok { + filteredBlobs = append(filteredBlobs, blob) + } + } + + return filteredBlobs, nil +} diff --git a/api/service/api_test.go b/api/service/api_test.go new file mode 100644 index 0000000..fed6807 --- /dev/null +++ b/api/service/api_test.go @@ -0,0 +1,253 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "testing" + + "github.com/attestantio/go-eth2-client/api" + v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/base-org/blob-archiver/api/metrics" + "github.com/base-org/blob-archiver/common/beacon/beacontest" + "github.com/base-org/blob-archiver/common/blobtest" + "github.com/base-org/blob-archiver/common/storage" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/stretchr/testify/require" +) + +func TestIsHash(t *testing.T) { + require.True(t, isHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")) + // Invalid hex character, ending with z + require.False(t, isHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdez")) + // Missing 0x prefix + require.False(t, isHash("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")) + + require.False(t, isHash("genesis")) + require.False(t, isHash("finalized")) + require.False(t, isHash("123")) // slot + require.False(t, isHash("unknown")) // incorrect input +} + +func TestIsSlot(t *testing.T) { + require.True(t, isSlot("123")) + require.False(t, isSlot("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")) + require.False(t, isSlot("genesis")) + require.False(t, isSlot("finalized")) + require.False(t, isSlot("unknown")) +} + +func TestIsNamedIdentifier(t *testing.T) { + require.True(t, isKnownIdentifier("genesis")) + require.True(t, isKnownIdentifier("finalized")) + require.False(t, isKnownIdentifier("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")) + require.False(t, isKnownIdentifier("123")) + require.False(t, isKnownIdentifier("unknown")) +} + +func setup(t *testing.T) (*API, *storage.FileStorage, *beacontest.StubBeaconClient, func()) { + logger := testlog.Logger(t, log.LvlInfo) + tempDir, err := os.MkdirTemp("", "test") + require.NoError(t, err) + fs := storage.NewFileStorage(tempDir, logger) + beacon := beacontest.NewEmptyStubBeaconClient() + r := opmetrics.NewRegistry() + m := metrics.NewMetrics(r) + api := NewAPI(fs, beacon, m, r, logger) + return api, fs, beacon, func() { + require.NoError(t, os.RemoveAll(tempDir)) + } +} + +func TestAPIService(t *testing.T) { + a, fs, beaconClient, cleanup := setup(t) + defer cleanup() + + rootOne := common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef") + rootTwo := common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890222222") + + blockOne := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: rootOne, + }, + BlobSidecars: storage.BlobSidecars{ + Data: blobtest.NewBlobSidecars(t, 2), + }, + } + + blockTwo := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: rootTwo, + }, + BlobSidecars: storage.BlobSidecars{ + Data: blobtest.NewBlobSidecars(t, 2), + }, + } + + err := fs.Write(context.Background(), blockOne) + require.NoError(t, err) + + err = fs.Write(context.Background(), blockTwo) + require.NoError(t, err) + + beaconClient.Headers["finalized"] = &v1.BeaconBlockHeader{ + Root: phase0.Root(rootOne), + } + + beaconClient.Headers["head"] = &v1.BeaconBlockHeader{ + Root: phase0.Root(rootTwo), + } + + beaconClient.Headers["1234"] = &v1.BeaconBlockHeader{ + Root: phase0.Root(rootTwo), + } + + tests := []struct { + name string + path string + status int + expected *storage.BlobSidecars + errMessage string + }{ + { + name: "fetch root one", + path: fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%s", rootOne), + status: 200, + expected: &blockOne.BlobSidecars, + }, + { + name: "fetch root two", + path: fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%s", rootTwo), + status: 200, + expected: &blockTwo.BlobSidecars, + }, + { + name: "fetch unknown", + path: "/eth/v1/beacon/blob_sidecars/0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abc111", + status: 404, + errMessage: "Block not found", + }, + { + name: "fetch head", + path: "/eth/v1/beacon/blob_sidecars/head", + status: 200, + expected: &blockTwo.BlobSidecars, + }, + { + name: "fetch finalized", + path: "/eth/v1/beacon/blob_sidecars/finalized", + status: 200, + expected: &blockOne.BlobSidecars, + }, + { + name: "fetch slot 1234", + path: "/eth/v1/beacon/blob_sidecars/1234", + status: 200, + expected: &blockTwo.BlobSidecars, + }, + { + name: "indices only returns requested indices", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=1", + status: 200, + expected: &storage.BlobSidecars{ + Data: []*deneb.BlobSidecar{ + blockTwo.BlobSidecars.Data[1], + }, + }, + }, + { + name: "indices out of bounds returns empty array", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=3", + status: 200, + expected: &storage.BlobSidecars{ + Data: []*deneb.BlobSidecar{}, + }, + }, + { + name: "no 0x on hash", + path: "/eth/v1/beacon/blob_sidecars/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + status: 400, + errMessage: "invalid block id: 1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + }, + { + name: "invalid hash", + path: "/eth/v1/beacon/blob_sidecars/0x1234567890abcdef123", + status: 400, + errMessage: "invalid block id: 0x1234567890abcdef123", + }, + { + name: "invalid named identifier", + path: "/eth/v1/beacon/blob_sidecars/foobar", + status: 400, + errMessage: "invalid block id: foobar", + }, + { + name: "invalid no parameter specified", + path: "/eth/v1/beacon/blob_sidecars/", + status: 404, + }, + { + name: "unknown route", + path: "/eth/v1/", + status: 404, + }, + } + + responseFormat := []string{"application/json", "application/octet-stream"} + + for _, test := range tests { + for _, rf := range responseFormat { + t.Run(fmt.Sprintf("%s-%s", test.name, rf), func(t *testing.T) { + request := httptest.NewRequest("GET", test.path, nil) + request.Header.Set("Accept", rf) + + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, test.status, response.Code) + + if test.status == 200 && test.expected != nil { + blobSidecars := storage.BlobSidecars{} + + var err error + if rf == "application/octet-stream" { + res := api.BlobSidecars{} + err = res.UnmarshalSSZ(response.Body.Bytes()) + blobSidecars.Data = res.Sidecars + } else { + err = json.Unmarshal(response.Body.Bytes(), &blobSidecars) + } + + require.NoError(t, err) + require.Equal(t, *test.expected, blobSidecars) + } else if test.status != 200 && rf == "application/json" && test.errMessage != "" { + var e httpError + err := json.Unmarshal(response.Body.Bytes(), &e) + require.NoError(t, err) + require.Equal(t, test.status, e.Code) + require.Equal(t, test.errMessage, e.Message) + } + }) + } + } +} + +func TestHealthHandler(t *testing.T) { + a, _, _, cleanup := setup(t) + defer cleanup() + + request := httptest.NewRequest("GET", "/healthz", nil) + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 200, response.Code) +} diff --git a/api/service/service.go b/api/service/service.go new file mode 100644 index 0000000..80d73f6 --- /dev/null +++ b/api/service/service.go @@ -0,0 +1,89 @@ +package service + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + + client "github.com/attestantio/go-eth2-client" + "github.com/base-org/blob-archiver/api/flags" + "github.com/base-org/blob-archiver/api/metrics" + "github.com/base-org/blob-archiver/common/storage" + "github.com/ethereum-optimism/optimism/op-service/httputil" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/log" + "github.com/prometheus/client_golang/prometheus" +) + +var ErrAlreadyStopped = errors.New("already stopped") + +func NewAPIService(l log.Logger, dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, cfg flags.APIConfig, registry *prometheus.Registry, m metrics.Metricer) *APIService { + router := NewAPI(dataStoreClient, beaconClient, m, registry, l) + return &APIService{ + log: l, + cfg: cfg, + registry: registry, + router: router, + } +} + +type APIService struct { + stopped atomic.Bool + log log.Logger + cfg flags.APIConfig + registry *prometheus.Registry + metricsServer *httputil.HTTPServer + apiServer *httputil.HTTPServer + router *API +} + +func (a *APIService) Start(ctx context.Context) error { + if a.cfg.MetricsConfig.Enabled { + a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort) + srv, err := opmetrics.StartServer(a.registry, a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort) + if err != nil { + return err + } + + a.log.Info("started metrics server", "addr", srv.Addr()) + a.metricsServer = srv + } + + a.log.Debug("starting API server", "address", a.cfg.ListenAddr) + + srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.router.router) + if err != nil { + return fmt.Errorf("failed to start API server: %w", err) + } + + a.log.Info("API server started", "address", srv.Addr().String()) + a.apiServer = srv + return nil +} + +func (a *APIService) Stop(ctx context.Context) error { + if a.stopped.Load() { + return ErrAlreadyStopped + } + a.log.Info("Stopping Archiver") + a.stopped.Store(true) + + if a.apiServer != nil { + if err := a.apiServer.Shutdown(ctx); err != nil { + return err + } + } + + if a.metricsServer != nil { + if err := a.metricsServer.Stop(ctx); err != nil { + return err + } + } + + return nil +} + +func (a *APIService) Stopped() bool { + return a.stopped.Load() +} diff --git a/archiver/Makefile b/archiver/Makefile new file mode 100644 index 0000000..bc72659 --- /dev/null +++ b/archiver/Makefile @@ -0,0 +1,13 @@ +blob-archiver: + env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-archiver ./cmd/main.go + +clean: + rm -f bin/blob-archiver + +test: + go test -v -race ./... + +.PHONY: \ + blob-archiver \ + clean \ + test \ No newline at end of file diff --git a/archiver/cmd/main.go b/archiver/cmd/main.go new file mode 100644 index 0000000..07e3849 --- /dev/null +++ b/archiver/cmd/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/base-org/blob-archiver/archiver/flags" + "github.com/base-org/blob-archiver/archiver/metrics" + "github.com/base-org/blob-archiver/archiver/service" + "github.com/base-org/blob-archiver/common/beacon" + "github.com/base-org/blob-archiver/common/storage" + opservice "github.com/ethereum-optimism/optimism/op-service" + "github.com/ethereum-optimism/optimism/op-service/cliapp" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + Version = "v0.0.1" + GitCommit = "" + GitDate = "" +) + +func main() { + oplog.SetupDefaults() + + app := cli.NewApp() + app.Flags = cliapp.ProtectFlags(flags.Flags) + app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "") + app.Name = "blob-archiver" + app.Usage = "Archiver service for Ethereum blobs" + app.Description = "Service for fetching blobs and archiving them to a datastore" + app.Action = cliapp.LifecycleCmd(Main()) + + err := app.Run(os.Args) + if err != nil { + log.Crit("Application failed", "message", err) + } +} + +// Main is the entrypoint into the Archiver. +// This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed archiver. +func Main() cliapp.LifecycleAction { + return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) { + cfg := flags.ReadConfig(cliCtx) + + if err := cfg.Check(); err != nil { + return nil, fmt.Errorf("invalid CLI flags: %w", err) + } + + l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig) + oplog.SetGlobalLogHandler(l.GetHandler()) + opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l) + + registry := opmetrics.NewRegistry() + m := metrics.NewMetrics(registry) + + beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig) + if err != nil { + return nil, err + } + + storageClient, err := storage.NewStorage(cfg.StorageConfig, l) + if err != nil { + return nil, err + } + + l.Info("Initializing Archiver Service") + return service.NewArchiverService(l, cfg, storageClient, beaconClient, m, registry) + } +} diff --git a/archiver/flags/config.go b/archiver/flags/config.go new file mode 100644 index 0000000..2e5504a --- /dev/null +++ b/archiver/flags/config.go @@ -0,0 +1,53 @@ +package flags + +import ( + "fmt" + "time" + + common "github.com/base-org/blob-archiver/common/flags" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + geth "github.com/ethereum/go-ethereum/common" + "github.com/urfave/cli/v2" +) + +type ArchiverConfig struct { + LogConfig oplog.CLIConfig + MetricsConfig opmetrics.CLIConfig + BeaconConfig common.BeaconConfig + StorageConfig common.StorageConfig + PollInterval time.Duration + OriginBlock geth.Hash +} + +func (c ArchiverConfig) Check() error { + if err := c.StorageConfig.Check(); err != nil { + return err + } + + if err := c.BeaconConfig.Check(); err != nil { + return err + } + + if c.PollInterval == 0 { + return fmt.Errorf("archiver poll interval must be set") + } + + if c.OriginBlock == (geth.Hash{}) { + return fmt.Errorf("invalid origin block") + } + + return nil +} + +func ReadConfig(cliCtx *cli.Context) ArchiverConfig { + pollInterval, _ := time.ParseDuration(cliCtx.String(ArchiverPollIntervalFlag.Name)) + return ArchiverConfig{ + LogConfig: oplog.ReadCLIConfig(cliCtx), + MetricsConfig: opmetrics.ReadCLIConfig(cliCtx), + BeaconConfig: common.NewBeaconConfig(cliCtx), + StorageConfig: common.NewStorageConfig(cliCtx), + PollInterval: pollInterval, + OriginBlock: geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)), + } +} diff --git a/archiver/flags/flags.go b/archiver/flags/flags.go new file mode 100644 index 0000000..7032cbc --- /dev/null +++ b/archiver/flags/flags.go @@ -0,0 +1,40 @@ +package flags + +import ( + common "github.com/base-org/blob-archiver/common/flags" + opservice "github.com/ethereum-optimism/optimism/op-service" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/urfave/cli/v2" +) + +const EnvVarPrefix = "BLOB_ARCHIVER" + +var ( + ArchiverPollIntervalFlag = &cli.StringFlag{ + Name: "archiver-poll-interval", + Usage: "The interval at which the archiver polls for new blobs", + EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "ARCHIVER_POLL_INTERVAL"), + Value: "6s", + } + ArchiverOriginBlock = &cli.StringFlag{ + Name: "archiver-origin-block", + Usage: "The lastest block hash that the archiver will walk back to", + Required: true, + EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "ORIGIN_BLOCK"), + } +) + +func init() { + var flags []cli.Flag + + flags = append(flags, common.CLIFlags(EnvVarPrefix)...) + flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...) + flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...) + flags = append(flags, ArchiverPollIntervalFlag, ArchiverOriginBlock) + + Flags = flags +} + +// Flags contains the list of configuration options available to the binary. +var Flags []cli.Flag diff --git a/archiver/metrics/metrics.go b/archiver/metrics/metrics.go new file mode 100644 index 0000000..b05f9ab --- /dev/null +++ b/archiver/metrics/metrics.go @@ -0,0 +1,49 @@ +package metrics + +import ( + "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlockSource string + +var ( + metricsNamespace = "blob_archiver" + + BlockSourceBackfill BlockSource = "backfill" + BlockSourceLive BlockSource = "live" +) + +type Metricer interface { + RecordProcessedBlock(source BlockSource) + RecordStoredBlobs(count int) +} + +type metricsRecorder struct { + blockProcessedCounter *prometheus.CounterVec + blobsStored prometheus.Counter +} + +func NewMetrics(registry *prometheus.Registry) Metricer { + factory := metrics.With(registry) + return &metricsRecorder{ + blockProcessedCounter: factory.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "blocks_processed", + Help: "number of times processing loop has run", + }, []string{"source"}), + blobsStored: factory.NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "blobs_stored", + Help: "number of blobs stored", + }), + } +} + +func (m *metricsRecorder) RecordStoredBlobs(count int) { + m.blobsStored.Add(float64(count)) +} + +func (m *metricsRecorder) RecordProcessedBlock(source BlockSource) { + m.blockProcessedCounter.WithLabelValues(string(source)).Inc() +} diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go new file mode 100644 index 0000000..d6a2a02 --- /dev/null +++ b/archiver/service/archiver.go @@ -0,0 +1,223 @@ +package service + +import ( + "context" + "errors" + "sync/atomic" + "time" + + client "github.com/attestantio/go-eth2-client" + "github.com/attestantio/go-eth2-client/api" + v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/base-org/blob-archiver/archiver/flags" + "github.com/base-org/blob-archiver/archiver/metrics" + "github.com/base-org/blob-archiver/common/storage" + "github.com/ethereum-optimism/optimism/op-service/httputil" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/prometheus/client_golang/prometheus" +) + +const backfillErrorRetryInterval = 5 * time.Second + +var ErrAlreadyStopped = errors.New("already stopped") + +type BeaconClient interface { + client.BlobSidecarsProvider + client.BeaconBlockHeadersProvider +} + +func NewArchiverService(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer, registry *prometheus.Registry) (*ArchiverService, error) { + return &ArchiverService{ + log: l, + cfg: cfg, + registry: registry, + dataStoreClient: dataStoreClient, + metrics: m, + stopCh: make(chan struct{}), + beaconClient: client, + }, nil +} + +type ArchiverService struct { + stopped atomic.Bool + stopCh chan struct{} + log log.Logger + dataStoreClient storage.DataStore + beaconClient BeaconClient + registry *prometheus.Registry + metricsServer *httputil.HTTPServer + cfg flags.ArchiverConfig + metrics metrics.Metricer +} + +func (a *ArchiverService) Start(ctx context.Context) error { + if a.cfg.MetricsConfig.Enabled { + a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort) + srv, err := opmetrics.StartServer(a.registry, a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort) + if err != nil { + return err + } + + a.log.Info("started metrics server", "addr", srv.Addr()) + a.metricsServer = srv + } + + currentBlob, _, err := a.persistBlobsForBlockToS3(ctx, "head") + if err != nil { + a.log.Error("failed to seed archiver with initial block", "err", err) + return err + } + + go a.backfillBlobs(ctx, currentBlob) + + return a.trackLatestBlocks(ctx) +} + +func (a *ArchiverService) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier string) (*v1.BeaconBlockHeader, bool, error) { + currentHeader, err := a.beaconClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{ + Block: blockIdentifier, + }) + + if err != nil { + a.log.Error("failed to fetch latest beacon block header", "err", err) + return nil, false, err + } + + exists, err := a.dataStoreClient.Exists(ctx, common.Hash(currentHeader.Data.Root)) + if err != nil { + a.log.Error("failed to check if blob exists", "err", err) + return nil, false, err + } + + if exists { + a.log.Debug("blob already exists", "hash", currentHeader.Data.Root) + return currentHeader.Data, true, nil + } + + blobSidecars, err := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{ + Block: currentHeader.Data.Root.String(), + }) + + if err != nil { + a.log.Error("failed to fetch blob sidecars", "err", err) + return nil, false, err + } + + a.log.Debug("fetched blob sidecars", "count", len(blobSidecars.Data)) + + blobData := storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: common.Hash(currentHeader.Data.Root), + }, + BlobSidecars: storage.BlobSidecars{Data: blobSidecars.Data}, + } + + err = a.dataStoreClient.Write(ctx, blobData) + + if err != nil { + a.log.Error("failed to write blob", "err", err) + return nil, false, err + } + + a.metrics.RecordStoredBlobs(len(blobSidecars.Data)) + + return currentHeader.Data, false, nil +} + +func (a *ArchiverService) Stop(ctx context.Context) error { + if a.stopped.Load() { + return ErrAlreadyStopped + } + a.log.Info("Stopping Archiver") + a.stopped.Store(true) + + close(a.stopCh) + + if a.metricsServer != nil { + if err := a.metricsServer.Stop(ctx); err != nil { + return err + } + } + + return nil +} + +func (a *ArchiverService) Stopped() bool { + return a.stopped.Load() +} + +func (a *ArchiverService) backfillBlobs(ctx context.Context, latest *v1.BeaconBlockHeader) { + current, alreadyExists, err := latest, false, error(nil) + + for !alreadyExists { + if common.Hash(current.Root) == a.cfg.OriginBlock { + a.log.Info("reached origin block", "hash", current.Root.String()) + return + } + + previous := current + current, alreadyExists, err = a.persistBlobsForBlockToS3(ctx, previous.Header.Message.ParentRoot.String()) + if err != nil { + a.log.Error("failed to persist blobs for block, will retry", "err", err, "hash", previous.Header.Message.ParentRoot.String()) + // Revert back to block we failed to fetch + current = previous + time.Sleep(backfillErrorRetryInterval) + continue + } + + if !alreadyExists { + a.metrics.RecordProcessedBlock(metrics.BlockSourceBackfill) + } + } + + a.log.Info("backfill complete", "endHash", current.Root.String(), "startHash", latest.Root.String()) +} + +func (a *ArchiverService) trackLatestBlocks(ctx context.Context) error { + t := time.NewTicker(a.cfg.PollInterval) + defer t.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-a.stopCh: + return nil + case <-t.C: + a.processBlocksUntilKnownBlock(ctx) + } + } +} + +func (a *ArchiverService) processBlocksUntilKnownBlock(ctx context.Context) { + a.log.Debug("refreshing live data") + + var start *v1.BeaconBlockHeader + currentBlockId := "head" + + for { + current, alreadyExisted, err := a.persistBlobsForBlockToS3(ctx, currentBlockId) + + if err != nil { + a.log.Error("failed to update live blobs for block", "err", err, "blockId", currentBlockId) + return + } + + if start == nil { + start = current + } + + if !alreadyExisted { + a.metrics.RecordProcessedBlock(metrics.BlockSourceLive) + } else { + a.log.Debug("blob already exists", "hash", current.Root.String()) + break + } + + currentBlockId = current.Header.Message.ParentRoot.String() + } + + a.log.Info("live data refreshed", "startHash", start.Root.String(), "endHash", currentBlockId) +} diff --git a/archiver/service/archiver_test.go b/archiver/service/archiver_test.go new file mode 100644 index 0000000..a7514d6 --- /dev/null +++ b/archiver/service/archiver_test.go @@ -0,0 +1,252 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/base-org/blob-archiver/archiver/flags" + "github.com/base-org/blob-archiver/archiver/metrics" + "github.com/base-org/blob-archiver/common/beacon/beacontest" + "github.com/base-org/blob-archiver/common/blobtest" + "github.com/base-org/blob-archiver/common/storage" + "github.com/base-org/blob-archiver/common/storage/storagetest" + "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func setup(t *testing.T, beacon *beacontest.StubBeaconClient) (*ArchiverService, *storagetest.TestFileStorage) { + l := testlog.Logger(t, log.LvlInfo) + fs := storagetest.NewTestFileStorage(t, l) + registry := prometheus.NewRegistry() + m := metrics.NewMetrics(registry) + svc, err := NewArchiverService(l, flags.ArchiverConfig{ + PollInterval: 5 * time.Second, + OriginBlock: blobtest.OriginBlock, + }, fs, beacon, m, registry) + require.NoError(t, err) + return svc, fs +} + +func TestArchiver_FetchAndPersist(t *testing.T) { + svc, fs := setup(t, beacontest.NewDefaultStubBeaconClient(t)) + + fs.CheckNotExistsOrFail(t, blobtest.OriginBlock) + + header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.OriginBlock.String()) + require.False(t, alreadyExists) + require.NoError(t, err) + require.NotNil(t, header) + require.Equal(t, blobtest.OriginBlock.String(), common.Hash(header.Root).String()) + + fs.CheckExistsOrFail(t, blobtest.OriginBlock) + + header, alreadyExists, err = svc.persistBlobsForBlockToS3(context.Background(), blobtest.OriginBlock.String()) + require.True(t, alreadyExists) + require.NoError(t, err) + require.NotNil(t, header) + require.Equal(t, blobtest.OriginBlock.String(), common.Hash(header.Root).String()) + + fs.CheckExistsOrFail(t, blobtest.OriginBlock) +} + +func TestArchiver_BackfillToOrigin(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // We have the current head, which is block 5 written to storage + err := fs.Write(context.Background(), storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.Five, + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.Five.String()], + }, + }) + require.NoError(t, err) + // We expect to backfill all blocks to the origin + expectedBlobs := []common.Hash{blobtest.Four, blobtest.Three, blobtest.Two, blobtest.One, blobtest.OriginBlock} + + for _, blob := range expectedBlobs { + fs.CheckNotExistsOrFail(t, blob) + } + + svc.backfillBlobs(context.Background(), beacon.Headers[blobtest.Five.String()]) + + for _, blob := range expectedBlobs { + fs.CheckExistsOrFail(t, blob) + data := fs.ReadOrFail(t, blob) + require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()]) + } +} + +func TestArchiver_BackfillToExistingBlock(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // We have the current head, which is block 5 written to storage + err := fs.Write(context.Background(), storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.Five, + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.Five.String()], + }, + }) + require.NoError(t, err) + + // We also have block 1 written to storage + err = fs.Write(context.Background(), storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.One, + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.One.String()], + }, + }) + require.NoError(t, err) + + // We expect to backfill all blobs between 5 and 1 + expectedBlobs := []common.Hash{blobtest.Four, blobtest.Three, blobtest.Two} + + for _, blob := range expectedBlobs { + exists, err := fs.Exists(context.Background(), blob) + require.NoError(t, err) + require.False(t, exists) + } + + svc.backfillBlobs(context.Background(), beacon.Headers[blobtest.Five.String()]) + + for _, blob := range expectedBlobs { + exists, err := fs.Exists(context.Background(), blob) + require.NoError(t, err) + require.True(t, exists) + + data, err := fs.Read(context.Background(), blob) + require.NoError(t, err) + require.NotNil(t, data) + require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()]) + } +} + +func TestArchiver_LatestStopsAtExistingBlock(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // 5 is the current head, if three already exists, we should write 5 and 4 and stop at three + fs.WriteOrFail(t, storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.Three, + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.Three.String()], + }, + }) + + fs.CheckNotExistsOrFail(t, blobtest.Five) + fs.CheckNotExistsOrFail(t, blobtest.Four) + fs.CheckExistsOrFail(t, blobtest.Three) + + svc.processBlocksUntilKnownBlock(context.Background()) + + fs.CheckExistsOrFail(t, blobtest.Five) + five := fs.ReadOrFail(t, blobtest.Five) + require.Equal(t, five.Header.BeaconBlockHash, blobtest.Five) + require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + + fs.CheckExistsOrFail(t, blobtest.Four) + four := fs.ReadOrFail(t, blobtest.Four) + require.Equal(t, four.Header.BeaconBlockHash, blobtest.Four) + require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) + + fs.CheckExistsOrFail(t, blobtest.Three) + three := fs.ReadOrFail(t, blobtest.Three) + require.Equal(t, three.Header.BeaconBlockHash, blobtest.Three) + require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()]) +} + +func TestArchiver_LatestNoNewData(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // 5 is the current head, if 5 already exists, this should be a no-op + fs.WriteOrFail(t, storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: common.Hash(beacon.Headers["head"].Root), + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.Three.String()], + }, + }) + + fs.CheckExistsOrFail(t, blobtest.Five) + fs.CheckNotExistsOrFail(t, blobtest.Four) + + svc.processBlocksUntilKnownBlock(context.Background()) + + fs.CheckExistsOrFail(t, blobtest.Five) + fs.CheckNotExistsOrFail(t, blobtest.Four) +} + +func TestArchiver_LatestConsumesNewBlocks(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // set current head to 4, and write four + beacon.Headers["head"] = beacon.Headers[blobtest.Four.String()] + fs.WriteOrFail(t, storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: common.Hash(beacon.Headers[blobtest.Four.String()].Root), + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.Four.String()], + }, + }) + + svc.processBlocksUntilKnownBlock(context.Background()) + + // No new data (5) is written and latest stops at known block (4), so 3 should not exist + fs.CheckNotExistsOrFail(t, blobtest.Five) + fs.CheckExistsOrFail(t, blobtest.Four) + fs.CheckNotExistsOrFail(t, blobtest.Three) + + // set current head to 5, and check it fetches new data + beacon.Headers["head"] = beacon.Headers[blobtest.Five.String()] + + svc.processBlocksUntilKnownBlock(context.Background()) + fs.CheckExistsOrFail(t, blobtest.Five) + fs.CheckExistsOrFail(t, blobtest.Four) + fs.CheckNotExistsOrFail(t, blobtest.Three) +} + +func TestArchiver_LatestStopsAtOrigin(t *testing.T) { + beacon := beacontest.NewDefaultStubBeaconClient(t) + svc, fs := setup(t, beacon) + + // 5 is the current head, if origin already exists, we should stop at origin + fs.WriteOrFail(t, storage.BlobData{ + Header: storage.Header{ + BeaconBlockHash: blobtest.OriginBlock, + }, + BlobSidecars: storage.BlobSidecars{ + Data: beacon.Blobs[blobtest.OriginBlock.String()], + }, + }) + + // Should write all blocks back to Origin + toWrite := []common.Hash{blobtest.Five, blobtest.Four, blobtest.Three, blobtest.Two, blobtest.One} + for _, hash := range toWrite { + fs.CheckNotExistsOrFail(t, hash) + } + + svc.processBlocksUntilKnownBlock(context.Background()) + + for _, hash := range toWrite { + fs.CheckExistsOrFail(t, hash) + data := fs.ReadOrFail(t, hash) + require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[hash.String()]) + } +} diff --git a/common/beacon/beacontest/stub.go b/common/beacon/beacontest/stub.go new file mode 100644 index 0000000..7e48440 --- /dev/null +++ b/common/beacon/beacontest/stub.go @@ -0,0 +1,86 @@ +package beacontest + +import ( + "context" + "fmt" + "testing" + + "github.com/attestantio/go-eth2-client/api" + v1 "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/base-org/blob-archiver/common/blobtest" + "github.com/ethereum/go-ethereum/common" +) + +type StubBeaconClient struct { + Headers map[string]*v1.BeaconBlockHeader + Blobs map[string][]*deneb.BlobSidecar +} + +func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.BeaconBlockHeaderOpts) (*api.Response[*v1.BeaconBlockHeader], error) { + header, found := s.Headers[opts.Block] + if !found { + return nil, fmt.Errorf("block not found") + } + return &api.Response[*v1.BeaconBlockHeader]{ + Data: header, + }, nil +} + +func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidecarsOpts) (*api.Response[[]*deneb.BlobSidecar], error) { + blobs, found := s.Blobs[opts.Block] + if !found { + return nil, fmt.Errorf("block not found") + } + return &api.Response[[]*deneb.BlobSidecar]{ + Data: blobs, + }, nil +} + +func NewEmptyStubBeaconClient() *StubBeaconClient { + return &StubBeaconClient{ + Headers: make(map[string]*v1.BeaconBlockHeader), + Blobs: make(map[string][]*deneb.BlobSidecar), + } +} + +func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient { + makeHeader := func(slot uint64, hash, parent common.Hash) *v1.BeaconBlockHeader { + return &v1.BeaconBlockHeader{ + Root: phase0.Root(hash), + Header: &phase0.SignedBeaconBlockHeader{ + Message: &phase0.BeaconBlockHeader{ + Slot: phase0.Slot(slot), + ParentRoot: phase0.Root(parent), + }, + }, + } + } + + headBlobs := blobtest.NewBlobSidecars(t, 6) + finalizedBlobs := blobtest.NewBlobSidecars(t, 4) + + return &StubBeaconClient{ + Headers: map[string]*v1.BeaconBlockHeader{ + blobtest.OriginBlock.String(): makeHeader(10, blobtest.OriginBlock, common.Hash{9, 9, 9}), + blobtest.One.String(): makeHeader(11, blobtest.One, blobtest.OriginBlock), + blobtest.Two.String(): makeHeader(12, blobtest.Two, blobtest.One), + blobtest.Three.String(): makeHeader(13, blobtest.Three, blobtest.Two), + blobtest.Four.String(): makeHeader(14, blobtest.Four, blobtest.Three), + blobtest.Five.String(): makeHeader(15, blobtest.Five, blobtest.Four), + "head": makeHeader(15, blobtest.Five, blobtest.Four), + "finalized": makeHeader(13, blobtest.Three, blobtest.Two), + }, + Blobs: map[string][]*deneb.BlobSidecar{ + blobtest.OriginBlock.String(): blobtest.NewBlobSidecars(t, 1), + blobtest.One.String(): blobtest.NewBlobSidecars(t, 2), + blobtest.Two.String(): blobtest.NewBlobSidecars(t, 0), + blobtest.Three.String(): finalizedBlobs, + blobtest.Four.String(): blobtest.NewBlobSidecars(t, 5), + blobtest.Five.String(): headBlobs, + "head": headBlobs, + "finalized": finalizedBlobs, + }, + } +} diff --git a/common/beacon/client.go b/common/beacon/client.go new file mode 100644 index 0000000..0715ca7 --- /dev/null +++ b/common/beacon/client.go @@ -0,0 +1,26 @@ +package beacon + +import ( + "context" + + client "github.com/attestantio/go-eth2-client" + "github.com/attestantio/go-eth2-client/http" + "github.com/base-org/blob-archiver/common/flags" +) + +type Client interface { + client.BeaconBlockHeadersProvider + client.BlobSidecarsProvider +} + +func NewBeaconClient(ctx context.Context, cfg flags.BeaconConfig) (Client, error) { + cctx, cancel := context.WithCancel(ctx) + defer cancel() + + c, err := http.New(cctx, http.WithAddress(cfg.BeaconUrl), http.WithTimeout(cfg.BeaconClientTimeout)) + if err != nil { + return nil, err + } + + return c.(*http.Service), nil +} diff --git a/common/blobtest/helpers.go b/common/blobtest/helpers.go new file mode 100644 index 0000000..fcb7e24 --- /dev/null +++ b/common/blobtest/helpers.go @@ -0,0 +1,47 @@ +package blobtest + +import ( + "crypto/rand" + "testing" + + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +var ( + OriginBlock = common.Hash{9, 9, 9, 9, 9} + One = common.Hash{1} + Two = common.Hash{2} + Three = common.Hash{3} + Four = common.Hash{4} + Five = common.Hash{5} +) + +func RandBytes(t *testing.T, size uint) []byte { + randomBytes := make([]byte, size) + _, err := rand.Read(randomBytes) + require.NoError(t, err) + return randomBytes +} + +func NewBlobSidecar(t *testing.T, i uint) *deneb.BlobSidecar { + return &deneb.BlobSidecar{ + Index: deneb.BlobIndex(i), + Blob: deneb.Blob(RandBytes(t, 131072)), + KZGCommitment: deneb.KZGCommitment(RandBytes(t, 48)), + KZGProof: deneb.KZGProof(RandBytes(t, 48)), + SignedBlockHeader: &phase0.SignedBeaconBlockHeader{ + Message: &phase0.BeaconBlockHeader{}, + }, + } +} + +func NewBlobSidecars(t *testing.T, count uint) []*deneb.BlobSidecar { + result := make([]*deneb.BlobSidecar, count) + for i := uint(0); i < count; i++ { + result[i] = NewBlobSidecar(t, i) + } + return result +} diff --git a/common/flags/config.go b/common/flags/config.go new file mode 100644 index 0000000..869092a --- /dev/null +++ b/common/flags/config.go @@ -0,0 +1,122 @@ +package flags + +import ( + "errors" + "time" + + "github.com/urfave/cli/v2" +) + +type DataStorage string + +const ( + DataStorageUnknown DataStorage = "unknown" + DataStorageS3 DataStorage = "s3" + DataStorageFile DataStorage = "file" +) + +type S3Config struct { + Endpoint string + AccessKey string + SecretAccessKey string + UseHttps bool + Bucket string +} + +func (c S3Config) check() error { + if c.Endpoint == "" { + return errors.New("s3 endpoint must be set") + } + + if c.AccessKey == "" { + return errors.New("s3 access key must be set") + } + + if c.SecretAccessKey == "" { + return errors.New("s3 secret access key must be set") + } + + if c.Bucket == "" { + return errors.New("s3 bucket must be set") + } + + return nil +} + +type BeaconConfig struct { + BeaconUrl string + BeaconClientTimeout time.Duration +} + +type StorageConfig struct { + DataStorageType DataStorage + S3Config S3Config + FileStorageDirectory string +} + +func NewBeaconConfig(cliCtx *cli.Context) BeaconConfig { + timeout, _ := time.ParseDuration(cliCtx.String(BeaconHttpClientTimeoutFlagName)) + + return BeaconConfig{ + BeaconUrl: cliCtx.String(BeaconHttpFlagName), + BeaconClientTimeout: timeout, + } +} + +func NewStorageConfig(cliCtx *cli.Context) StorageConfig { + return StorageConfig{ + DataStorageType: toDataStorage(cliCtx.String(DataStoreFlagName)), + S3Config: readS3Config(cliCtx), + FileStorageDirectory: cliCtx.String(FileStorageDirectoryFlagName), + } +} + +func toDataStorage(s string) DataStorage { + if s == string(DataStorageS3) { + return DataStorageS3 + } + + if s == string(DataStorageFile) { + return DataStorageFile + } + + return DataStorageUnknown +} + +func readS3Config(ctx *cli.Context) S3Config { + return S3Config{ + Endpoint: ctx.String(S3EndpointFlagName), + AccessKey: ctx.String(S3AccessKeyFlagName), + SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName), + UseHttps: ctx.Bool(S3EndpointHttpsFlagName), + Bucket: ctx.String(S3BucketFlagName), + } +} + +func (c BeaconConfig) Check() error { + if c.BeaconUrl == "" { + return errors.New("beacon url must be set") + } + + if c.BeaconClientTimeout == 0 { + return errors.New("beacon client timeout must be set") + } + + return nil +} + +func (c StorageConfig) Check() error { + if c.DataStorageType == DataStorageUnknown { + return errors.New("unknown data-storage type") + } + + if c.DataStorageType == DataStorageS3 { + if err := c.S3Config.check(); err != nil { + return err + } + } else if c.DataStorageType == DataStorageFile && c.FileStorageDirectory == "" { + return errors.New("file storage directory must be set") + } + + return nil +} diff --git a/common/flags/flags.go b/common/flags/flags.go new file mode 100644 index 0000000..86a09d4 --- /dev/null +++ b/common/flags/flags.go @@ -0,0 +1,80 @@ +package flags + +import ( + opservice "github.com/ethereum-optimism/optimism/op-service" + "github.com/urfave/cli/v2" +) + +const ( + BeaconHttpFlagName = "l1-beacon-http" + BeaconHttpClientTimeoutFlagName = "l1-beacon-client-timeout" + DataStoreFlagName = "data-store" + S3EndpointFlagName = "s3-endpoint" + S3EndpointHttpsFlagName = "s3-endpoint-https" + S3AccessKeyFlagName = "s3-access-key" + S3SecretAccessKeyFlagName = "s3-secret-access-key" + S3BucketFlagName = "s3-bucket" + FileStorageDirectoryFlagName = "file-directory" +) + +func CLIFlags(envPrefix string) []cli.Flag { + return []cli.Flag{ + // Required Flags + &cli.StringFlag{ + Name: BeaconHttpFlagName, + Usage: "HTTP provider URL for L1 Beacon-node API", + Required: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "L1_BEACON_HTTP"), + }, + &cli.StringFlag{ + Name: DataStoreFlagName, + Usage: "The type of data-store, options are [s3, file]", + Required: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "DATA_STORE"), + }, + // Optional Flags + // S3 Data Store Flags + &cli.StringFlag{ + Name: S3EndpointFlagName, + Usage: "The URL for the S3 bucket (without the scheme http or https specified)", + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ENDPOINT"), + }, + &cli.BoolFlag{ + Name: S3EndpointHttpsFlagName, + Usage: "Whether to use https for the S3 bucket", + Value: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ENDPOINT_HTTPS"), + }, + &cli.StringFlag{ + Name: S3AccessKeyFlagName, + Usage: "The S3 access key for the bucket", + Hidden: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ACCESS_KEY"), + }, + &cli.StringFlag{ + Name: S3SecretAccessKeyFlagName, + Usage: "The S3 secret access key for the bucket", + Hidden: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_SECRET_ACCESS_KEY"), + }, + &cli.StringFlag{ + Name: S3BucketFlagName, + Usage: "The bucket to use", + Hidden: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_BUCKET"), + }, + // File Data Store Flags + &cli.StringFlag{ + Name: FileStorageDirectoryFlagName, + Usage: "The path to the directory to use for storing blobs on the file system", + EnvVars: opservice.PrefixEnvVar(envPrefix, "FILE_DIRECTORY"), + }, + // Beacon Client Settings + &cli.StringFlag{ + Name: BeaconHttpClientTimeoutFlagName, + Usage: "The timeout duration for the beacon client", + Value: "10s", + EnvVars: opservice.PrefixEnvVar(envPrefix, "L1_BEACON_CLIENT_TIMEOUT"), + }, + } +} diff --git a/common/storage/file.go b/common/storage/file.go new file mode 100644 index 0000000..990b8cf --- /dev/null +++ b/common/storage/file.go @@ -0,0 +1,72 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "path" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +type FileStorage struct { + log log.Logger + directory string +} + +func NewFileStorage(dir string, l log.Logger) *FileStorage { + return &FileStorage{ + log: l, + directory: dir, + } +} + +func (s *FileStorage) Exists(_ context.Context, hash common.Hash) (bool, error) { + _, err := os.Stat(s.fileName(hash)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *FileStorage) Read(_ context.Context, hash common.Hash) (BlobData, error) { + data, err := os.ReadFile(s.fileName(hash)) + if err != nil { + if os.IsNotExist(err) { + return BlobData{}, ErrNotFound + } + + return BlobData{}, err + } + var result BlobData + err = json.Unmarshal(data, &result) + if err != nil { + s.log.Warn("error decoding blob", "err", err, "hash", hash.String()) + return BlobData{}, ErrEncoding + } + return result, nil +} + +func (s *FileStorage) Write(_ context.Context, data BlobData) error { + b, err := json.Marshal(data) + if err != nil { + s.log.Warn("error encoding blob", "err", err) + return ErrEncoding + } + err = os.WriteFile(s.fileName(data.Header.BeaconBlockHash), b, 0644) + if err != nil { + s.log.Warn("error writing blob", "err", err) + return err + } + + s.log.Info("wrote blob", "hash", data.Header.BeaconBlockHash.String()) + return nil +} + +func (s *FileStorage) fileName(hash common.Hash) string { + return path.Join(s.directory, hash.String()) +} diff --git a/common/storage/file_test.go b/common/storage/file_test.go new file mode 100644 index 0000000..3f3ed03 --- /dev/null +++ b/common/storage/file_test.go @@ -0,0 +1,115 @@ +package storage + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/stretchr/testify/require" +) + +func setup(t *testing.T) (*FileStorage, func()) { + logger := testlog.Logger(t, log.LvlInfo) + tempDir, err := os.MkdirTemp("", "test") + require.NoError(t, err) + fs := NewFileStorage(tempDir, logger) + return fs, func() { + require.NoError(t, os.RemoveAll(tempDir)) + } +} + +func runTestExists(t *testing.T, s DataStore) { + id := common.Hash{1, 2, 3} + + exists, err := s.Exists(context.Background(), id) + require.NoError(t, err) + require.False(t, exists) + + err = s.Write(context.Background(), BlobData{ + Header: Header{ + BeaconBlockHash: id, + }, + BlobSidecars: BlobSidecars{}, + }) + require.NoError(t, err) + + exists, err = s.Exists(context.Background(), id) + require.NoError(t, err) + require.True(t, exists) +} + +func TestExists(t *testing.T) { + fs, cleanup := setup(t) + defer cleanup() + + runTestExists(t, fs) +} + +func runTestRead(t *testing.T, s DataStore) { + id := common.Hash{1, 2, 3} + + _, err := s.Read(context.Background(), id) + require.Error(t, err) + require.True(t, errors.Is(err, ErrNotFound)) + + err = s.Write(context.Background(), BlobData{ + Header: Header{ + BeaconBlockHash: id, + }, + BlobSidecars: BlobSidecars{}, + }) + require.NoError(t, err) + + data, err := s.Read(context.Background(), id) + require.NoError(t, err) + require.Equal(t, id, data.Header.BeaconBlockHash) +} + +func TestRead(t *testing.T) { + fs, cleanup := setup(t) + defer cleanup() + + runTestRead(t, fs) +} + +func TestBrokenStorage(t *testing.T) { + fs, cleanup := setup(t) + + id := common.Hash{1, 2, 3} + + // Delete the directory to simulate broken storage + cleanup() + + _, err := fs.Read(context.Background(), id) + require.Error(t, err) + + exists, err := fs.Exists(context.Background(), id) + require.False(t, exists) + require.NoError(t, err) // No error should be returned, as in this test we've just delted the directory + + err = fs.Write(context.Background(), BlobData{ + Header: Header{ + BeaconBlockHash: id, + }, + BlobSidecars: BlobSidecars{}, + }) + require.Error(t, err) +} + +func TestReadInvalidData(t *testing.T) { + fs, cleanup := setup(t) + defer cleanup() + + id := common.Hash{1, 2, 3} + + err := os.WriteFile(fs.fileName(id), []byte("invalid json"), 0644) + require.NoError(t, err) + + _, err = fs.Read(context.Background(), id) + require.Error(t, err) + require.True(t, errors.Is(err, ErrEncoding)) +} diff --git a/common/storage/s3.go b/common/storage/s3.go new file mode 100644 index 0000000..c5e4bf7 --- /dev/null +++ b/common/storage/s3.go @@ -0,0 +1,100 @@ +package storage + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/base-org/blob-archiver/common/flags" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type S3Storage struct { + s3 *minio.Client + bucket string + log log.Logger +} + +func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) { + client, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, ""), + Secure: cfg.UseHttps, + }) + + if err != nil { + return nil, err + } + + return &S3Storage{ + s3: client, + bucket: cfg.Bucket, + log: l, + }, nil +} + +func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) { + _, err := s.s3.StatObject(ctx, s.bucket, hash.String(), minio.StatObjectOptions{}) + if err != nil { + errResponse := minio.ToErrorResponse(err) + if errResponse.Code == "NoSuchKey" { + return false, nil + } else { + return false, err + } + } + + return true, nil +} + +func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error) { + res, err := s.s3.GetObject(ctx, s.bucket, hash.String(), minio.GetObjectOptions{}) + if err != nil { + s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err) + return BlobData{}, ErrStorage + } + defer res.Close() + _, err = res.Stat() + if err != nil { + errResponse := minio.ToErrorResponse(err) + if errResponse.Code == "NoSuchKey" { + s.log.Info("unable to find blob", "hash", hash.String()) + return BlobData{}, ErrNotFound + } else { + s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err) + return BlobData{}, ErrStorage + } + } + + var data BlobData + err = json.NewDecoder(res).Decode(&data) + if err != nil { + s.log.Warn("error decoding blob", "hash", hash.String(), "err", err) + return BlobData{}, ErrEncoding + } + + return data, nil +} + +func (s *S3Storage) Write(ctx context.Context, data BlobData) error { + b, err := json.Marshal(data) + if err != nil { + s.log.Warn("error encoding blob", "err", err) + return ErrEncoding + } + + reader := bytes.NewReader(b) + _, err = s.s3.PutObject(ctx, s.bucket, data.Header.BeaconBlockHash.String(), reader, int64(len(b)), minio.PutObjectOptions{ + ContentType: "application/json", + }) + + if err != nil { + s.log.Warn("error writing blob", "err", err) + return ErrStorage + } + + s.log.Info("wrote blob", "hash", data.Header.BeaconBlockHash.String()) + return nil +} diff --git a/common/storage/s3_test.go b/common/storage/s3_test.go new file mode 100644 index 0000000..9daa894 --- /dev/null +++ b/common/storage/s3_test.go @@ -0,0 +1,55 @@ +package storage + +import ( + "context" + "os" + "testing" + + "github.com/base-org/blob-archiver/common/flags" + "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum/log" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/require" +) + +// Prior to running these tests, a local Minio server must be running. +// You can accomplish this with: +// docker-compose down # shut down any running services +// docker-compose up minio create-buckets # start the minio service +func setupS3(t *testing.T) *S3Storage { + if os.Getenv("RUN_INTEGRATION_TESTS") == "" { + t.Skip("skipping integration tests: set RUN_INTEGRATION_TESTS environment variable") + } + + l := testlog.Logger(t, log.LvlInfo) + + s3, err := NewS3Storage(flags.S3Config{ + Endpoint: "localhost:9000", + AccessKey: "admin", + SecretAccessKey: "password", + UseHttps: false, + Bucket: "blobs", + }, l) + + require.NoError(t, err) + + for object := range s3.s3.ListObjects(context.Background(), "blobs", minio.ListObjectsOptions{}) { + err = s3.s3.RemoveObject(context.Background(), "blobs", object.Key, minio.RemoveObjectOptions{}) + require.NoError(t, err) + } + + require.NoError(t, err) + return s3 +} + +func TestS3Exists(t *testing.T) { + s3 := setupS3(t) + + runTestExists(t, s3) +} + +func TestS3Read(t *testing.T) { + s3 := setupS3(t) + + runTestRead(t, s3) +} diff --git a/common/storage/storage.go b/common/storage/storage.go new file mode 100644 index 0000000..6db3f0a --- /dev/null +++ b/common/storage/storage.go @@ -0,0 +1,78 @@ +package storage + +import ( + "context" + "errors" + + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/base-org/blob-archiver/common/flags" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const ( + blobSidecarSize = 131928 +) + +var ( + ErrNotFound = errors.New("blob not found") + ErrStorage = errors.New("error accessing storage") + ErrEncoding = errors.New("error encoding/decoding blob") +) + +type Header struct { + BeaconBlockHash common.Hash `json:"beacon_block_hash"` +} + +type BlobSidecars struct { + Data []*deneb.BlobSidecar `json:"data"` +} + +func (b *BlobSidecars) MarshalSSZ() ([]byte, error) { + result := make([]byte, b.SizeSSZ()) + + for i, sidecar := range b.Data { + sidecarBytes, err := sidecar.MarshalSSZ() + if err != nil { + return nil, err + } + + from := i * len(sidecarBytes) + to := (i + 1) * len(sidecarBytes) + + copy(result[from:to], sidecarBytes) + } + + return result, nil +} + +func (b *BlobSidecars) SizeSSZ() int { + return len(b.Data) * blobSidecarSize +} + +type BlobData struct { + Header Header `json:"header"` + BlobSidecars BlobSidecars `json:"blob_sidecars"` +} + +type DataStoreReader interface { + Exists(ctx context.Context, hash common.Hash) (bool, error) + Read(ctx context.Context, hash common.Hash) (BlobData, error) +} + +type DataStoreWriter interface { + Write(ctx context.Context, data BlobData) error +} + +type DataStore interface { + DataStoreReader + DataStoreWriter +} + +func NewStorage(cfg flags.StorageConfig, l log.Logger) (DataStore, error) { + if cfg.DataStorageType == flags.DataStorageS3 { + return NewS3Storage(cfg.S3Config, l) + } else { + return NewFileStorage(cfg.FileStorageDirectory, l), nil + } +} diff --git a/common/storage/storage_test.go b/common/storage/storage_test.go new file mode 100644 index 0000000..2803fe7 --- /dev/null +++ b/common/storage/storage_test.go @@ -0,0 +1,44 @@ +package storage + +import ( + "testing" + + "github.com/attestantio/go-eth2-client/api" + "github.com/attestantio/go-eth2-client/spec/deneb" + "github.com/base-org/blob-archiver/common/blobtest" + "github.com/stretchr/testify/require" +) + +func TestMarshalSSZ(t *testing.T) { + b := &BlobSidecars{ + Data: []*deneb.BlobSidecar{ + { + Index: 1, + Blob: deneb.Blob(blobtest.RandBytes(t, 131072)), + KZGCommitment: deneb.KZGCommitment(blobtest.RandBytes(t, 48)), + KZGProof: deneb.KZGProof(blobtest.RandBytes(t, 48)), + }, + { + Index: 2, + Blob: deneb.Blob(blobtest.RandBytes(t, 131072)), + KZGCommitment: deneb.KZGCommitment(blobtest.RandBytes(t, 48)), + KZGProof: deneb.KZGProof(blobtest.RandBytes(t, 48)), + }, + }, + } + + data, err := b.MarshalSSZ() + require.NoError(t, err) + + sidecars := api.BlobSidecars{} + err = sidecars.UnmarshalSSZ(data) + require.NoError(t, err) + + require.Equal(t, len(b.Data), len(sidecars.Sidecars)) + for i := range b.Data { + require.Equal(t, b.Data[i].Index, sidecars.Sidecars[i].Index) + require.Equal(t, b.Data[i].Blob, sidecars.Sidecars[i].Blob) + require.Equal(t, b.Data[i].KZGCommitment, sidecars.Sidecars[i].KZGCommitment) + require.Equal(t, b.Data[i].KZGProof, sidecars.Sidecars[i].KZGProof) + } +} diff --git a/common/storage/storagetest/stub.go b/common/storage/storagetest/stub.go new file mode 100644 index 0000000..ebef425 --- /dev/null +++ b/common/storage/storagetest/stub.go @@ -0,0 +1,46 @@ +package storagetest + +import ( + "context" + "testing" + + "github.com/base-org/blob-archiver/common/storage" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/stretchr/testify/require" +) + +type TestFileStorage struct { + *storage.FileStorage +} + +func NewTestFileStorage(t *testing.T, l log.Logger) *TestFileStorage { + dir := t.TempDir() + return &TestFileStorage{ + FileStorage: storage.NewFileStorage(dir, l), + } +} + +func (fs *TestFileStorage) CheckExistsOrFail(t *testing.T, hash common.Hash) { + exists, err := fs.Exists(context.Background(), hash) + require.NoError(t, err) + require.True(t, exists) +} + +func (fs *TestFileStorage) CheckNotExistsOrFail(t *testing.T, hash common.Hash) { + exists, err := fs.Exists(context.Background(), hash) + require.NoError(t, err) + require.False(t, exists) +} + +func (fs *TestFileStorage) WriteOrFail(t *testing.T, data storage.BlobData) { + err := fs.Write(context.Background(), data) + require.NoError(t, err) +} + +func (fs *TestFileStorage) ReadOrFail(t *testing.T, hash common.Hash) storage.BlobData { + data, err := fs.Read(context.Background(), hash) + require.NoError(t, err) + require.NotNil(t, data) + return data +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9fba628 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,46 @@ +version: "3.7" + +services: + api: + build: + context: . + dockerfile: Dockerfile + env_file: + - .env + command: + - "blob-api" + depends_on: + - minio + - create-buckets + archiver: + build: + context: . + dockerfile: Dockerfile + command: + - "blob-archiver" + env_file: + - .env + depends_on: + - minio + - create-buckets + minio: + restart: unless-stopped + image: minio/minio:latest + ports: + - "9000:9000" + - "9999:9999" + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + entrypoint: minio server /data --console-address ":9999" + create-buckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set minio http://minio:9000 admin password; + /usr/bin/mc mb minio/blobs; + /usr/bin/mc policy set public minio/blobs; + exit 0; + " \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..aeb1904 --- /dev/null +++ b/go.mod @@ -0,0 +1,109 @@ +module github.com/base-org/blob-archiver + +go 1.21.6 + +require ( + github.com/attestantio/go-eth2-client v0.19.10 + github.com/ethereum-optimism/optimism v1.4.0-rc.3 + github.com/ethereum/go-ethereum v1.13.5 + github.com/go-chi/chi/v5 v5.0.10 + github.com/minio/minio-go/v7 v7.0.66 + github.com/prometheus/client_golang v1.17.0 + github.com/stretchr/testify v1.8.4 + github.com/urfave/cli/v2 v2.25.7 +) + +require ( + github.com/DataDog/zstd v1.5.2 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/ferranbt/fastssz v0.1.3 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/goccy/go-yaml v1.9.2 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/holiman/uint256 v1.2.4 // indirect + github.com/huandu/go-clone v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect + github.com/r3labs/sse/v2 v2.10.0 // indirect + github.com/rivo/uniseg v0.4.3 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/rs/zerolog v1.29.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/supranational/blst v0.3.11 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bc49da5 --- /dev/null +++ b/go.sum @@ -0,0 +1,421 @@ +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/attestantio/go-eth2-client v0.19.10 h1:NLs9mcBvZpBTZ3du7Ey2NHQoj8d3UePY7pFBXX6C6qs= +github.com/attestantio/go-eth2-client v0.19.10/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/btcsuite/btcd v0.23.3 h1:4KH/JKy9WiCd+iUS9Mu0Zp7Dnj17TGdKrg9xc/FGj24= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.3 h1:SDlJ7bAm4ewvrmZtR0DaiYbQGdKPeaaIm7bM+qRhFeU= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.3/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4 h1:PuHFhOUMnD62r80dN+Ik5qco2drekgsUSVdcHsvllec= +github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ethereum-optimism/optimism v1.4.0-rc.3 h1:Lh5Ipb8DYGaovZ/mwuOiDz22K2LDE0cOCxcR7cUUQjw= +github.com/ethereum-optimism/optimism v1.4.0-rc.3/go.mod h1:O7kjR6x4dCRs+JYx5gMLk7CRWbbYAAknsd2qCrDE+HQ= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.13.5 h1:U6TCRciCqZRe4FPXmy1sMGxTfuk8P7u2UoinF3VbaFk= +github.com/ethereum/go-ethereum v1.13.5/go.mod h1:yMTu38GSuyxaYzQMViqNmQ1s3cE84abZexQmTgenWk0= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= +github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/fjl/memsize v0.0.1 h1:+zhkb+dhUgx0/e+M8sF0QqiouvMQUiKR+QYvdxIOKcQ= +github.com/fjl/memsize v0.0.1/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= +github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= +github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/goccy/go-yaml v1.9.2 h1:2Njwzw+0+pjU2gb805ZC1B/uBuAs2VcZ3K+ZgHwDs7w= +github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY= +github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE= +github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= +github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc= +github.com/huandu/go-clone v1.6.0/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= +github.com/huandu/go-clone/generic v1.6.0 h1:Wgmt/fUZ28r16F2Y3APotFD59sHk1p78K0XLdbUYN5U= +github.com/huandu/go-clone/generic v1.6.0/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= +github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= +github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= +github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= +github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= +github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= +github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10/go.mod h1:x/Pa0FF5Te9kdrlZKJK82YmAkvL8+f989USgz6Jiw7M= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= +gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= From fb94a5e3c1d8c0938d61753a66dfb593fba21384 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Sun, 11 Feb 2024 12:36:22 -0600 Subject: [PATCH 02/10] Add test for fetching duplicate indices --- api/service/api.go | 4 ++-- api/service/api_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api/service/api.go b/api/service/api.go index d6c707e..56d62a6 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -218,14 +218,14 @@ func filterBlobs(blobs []*deneb.BlobSidecar, indices string) ([]*deneb.BlobSidec return blobs, nil } - indicesMap := map[deneb.BlobIndex]bool{} + indicesMap := map[deneb.BlobIndex]struct{}{} for _, index := range splits { parsedInt, err := strconv.ParseUint(index, 10, 64) if err != nil { return nil, newIndicesError(index) } blobIndex := deneb.BlobIndex(parsedInt) - indicesMap[blobIndex] = true + indicesMap[blobIndex] = struct{}{} } filteredBlobs := make([]*deneb.BlobSidecar, 0) diff --git a/api/service/api_test.go b/api/service/api_test.go index fed6807..d302290 100644 --- a/api/service/api_test.go +++ b/api/service/api_test.go @@ -162,6 +162,16 @@ func TestAPIService(t *testing.T) { }, }, }, + { + name: "deduplicates indices", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=1,1,1", + status: 200, + expected: &storage.BlobSidecars{ + Data: []*deneb.BlobSidecar{ + blockTwo.BlobSidecars.Data[1], + }, + }, + }, { name: "indices out of bounds returns empty array", path: "/eth/v1/beacon/blob_sidecars/1234?indices=3", From a4cb75753047e96e80231eccf8f8ecbe00b6084e Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Sun, 11 Feb 2024 12:56:37 -0600 Subject: [PATCH 03/10] Add healthcheck to archiver service --- api/cmd/main.go | 7 +++-- api/metrics/metrics.go | 11 +++++++- api/service/api.go | 5 ++-- api/service/api_test.go | 8 +++--- api/service/service.go | 12 +++------ archiver/cmd/main.go | 6 ++--- archiver/metrics/metrics.go | 17 +++++++++--- archiver/service/api.go | 45 +++++++++++++++++++++++++++++++ archiver/service/api_test.go | 28 +++++++++++++++++++ archiver/service/archiver.go | 7 ++--- archiver/service/archiver_test.go | 8 +++--- 11 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 archiver/service/api.go create mode 100644 archiver/service/api_test.go diff --git a/api/cmd/main.go b/api/cmd/main.go index e736050..70c15be 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -13,7 +13,6 @@ import ( opservice "github.com/ethereum-optimism/optimism/op-service" "github.com/ethereum-optimism/optimism/op-service/cliapp" oplog "github.com/ethereum-optimism/optimism/op-service/log" - opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/log" "github.com/urfave/cli/v2" ) @@ -54,8 +53,7 @@ func Main() cliapp.LifecycleAction { oplog.SetGlobalLogHandler(l.GetHandler()) opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l) - registry := opmetrics.NewRegistry() - m := metrics.NewMetrics(registry) + m := metrics.NewMetrics() storageClient, err := storage.NewStorage(cfg.StorageConfig, l) if err != nil { @@ -68,6 +66,7 @@ func Main() cliapp.LifecycleAction { } l.Info("Initializing API Service") - return service.NewAPIService(l, storageClient, beaconClient, cfg, registry, m), nil + api := service.NewAPI(storageClient, beaconClient, m, l) + return service.NewService(l, api, cfg, m.Registry()), nil } } diff --git a/api/metrics/metrics.go b/api/metrics/metrics.go index 3a45da0..aa26d17 100644 --- a/api/metrics/metrics.go +++ b/api/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "github.com/ethereum-optimism/optimism/op-service/metrics" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -16,16 +17,20 @@ var ( ) type Metricer interface { + Registry() *prometheus.Registry RecordBlockIdType(t BlockIdType) } type metricsRecorder struct { inputType *prometheus.CounterVec + registry *prometheus.Registry } -func NewMetrics(registry *prometheus.Registry) Metricer { +func NewMetrics() Metricer { + registry := opmetrics.NewRegistry() factory := metrics.With(registry) return &metricsRecorder{ + registry: registry, inputType: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: MetricsNamespace, Name: "block_id_type", @@ -37,3 +42,7 @@ func NewMetrics(registry *prometheus.Registry) Metricer { func (m *metricsRecorder) RecordBlockIdType(t BlockIdType) { m.inputType.WithLabelValues(string(t)).Inc() } + +func (m *metricsRecorder) Registry() *prometheus.Registry { + return m.registry +} diff --git a/api/service/api.go b/api/service/api.go index 56d62a6..eddfc97 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/prometheus/client_golang/prometheus" ) type httpError struct { @@ -77,7 +76,7 @@ type API struct { metrics m.Metricer } -func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, metrics m.Metricer, registry *prometheus.Registry, logger log.Logger) *API { +func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, metrics m.Metricer, logger log.Logger) *API { result := &API{ dataStoreClient: dataStoreClient, beaconClient: beaconClient, @@ -92,7 +91,7 @@ func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconB r.Use(middleware.Recoverer) r.Use(middleware.Heartbeat("/healthz")) - recorder := opmetrics.NewPromHTTPRecorder(registry, m.MetricsNamespace) + recorder := opmetrics.NewPromHTTPRecorder(metrics.Registry(), m.MetricsNamespace) r.Use(func(handler http.Handler) http.Handler { return opmetrics.NewHTTPRecordingMiddleware(recorder, handler) }) diff --git a/api/service/api_test.go b/api/service/api_test.go index d302290..e597e9c 100644 --- a/api/service/api_test.go +++ b/api/service/api_test.go @@ -16,7 +16,6 @@ import ( "github.com/base-org/blob-archiver/common/beacon/beacontest" "github.com/base-org/blob-archiver/common/blobtest" "github.com/base-org/blob-archiver/common/storage" - opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum-optimism/optimism/op-service/testlog" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -58,10 +57,9 @@ func setup(t *testing.T) (*API, *storage.FileStorage, *beacontest.StubBeaconClie require.NoError(t, err) fs := storage.NewFileStorage(tempDir, logger) beacon := beacontest.NewEmptyStubBeaconClient() - r := opmetrics.NewRegistry() - m := metrics.NewMetrics(r) - api := NewAPI(fs, beacon, m, r, logger) - return api, fs, beacon, func() { + m := metrics.NewMetrics() + a := NewAPI(fs, beacon, m, logger) + return a, fs, beacon, func() { require.NoError(t, os.RemoveAll(tempDir)) } } diff --git a/api/service/service.go b/api/service/service.go index 80d73f6..84bc963 100644 --- a/api/service/service.go +++ b/api/service/service.go @@ -6,10 +6,7 @@ import ( "fmt" "sync/atomic" - client "github.com/attestantio/go-eth2-client" "github.com/base-org/blob-archiver/api/flags" - "github.com/base-org/blob-archiver/api/metrics" - "github.com/base-org/blob-archiver/common/storage" "github.com/ethereum-optimism/optimism/op-service/httputil" opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/log" @@ -18,13 +15,12 @@ import ( var ErrAlreadyStopped = errors.New("already stopped") -func NewAPIService(l log.Logger, dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, cfg flags.APIConfig, registry *prometheus.Registry, m metrics.Metricer) *APIService { - router := NewAPI(dataStoreClient, beaconClient, m, registry, l) +func NewService(l log.Logger, api *API, cfg flags.APIConfig, registry *prometheus.Registry) *APIService { return &APIService{ log: l, cfg: cfg, registry: registry, - router: router, + api: api, } } @@ -35,7 +31,7 @@ type APIService struct { registry *prometheus.Registry metricsServer *httputil.HTTPServer apiServer *httputil.HTTPServer - router *API + api *API } func (a *APIService) Start(ctx context.Context) error { @@ -52,7 +48,7 @@ func (a *APIService) Start(ctx context.Context) error { a.log.Debug("starting API server", "address", a.cfg.ListenAddr) - srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.router.router) + srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.api.router) if err != nil { return fmt.Errorf("failed to start API server: %w", err) } diff --git a/archiver/cmd/main.go b/archiver/cmd/main.go index 07e3849..4915e9d 100644 --- a/archiver/cmd/main.go +++ b/archiver/cmd/main.go @@ -13,7 +13,6 @@ import ( opservice "github.com/ethereum-optimism/optimism/op-service" "github.com/ethereum-optimism/optimism/op-service/cliapp" oplog "github.com/ethereum-optimism/optimism/op-service/log" - opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/log" "github.com/urfave/cli/v2" ) @@ -55,8 +54,7 @@ func Main() cliapp.LifecycleAction { oplog.SetGlobalLogHandler(l.GetHandler()) opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l) - registry := opmetrics.NewRegistry() - m := metrics.NewMetrics(registry) + m := metrics.NewMetrics() beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig) if err != nil { @@ -69,6 +67,6 @@ func Main() cliapp.LifecycleAction { } l.Info("Initializing Archiver Service") - return service.NewArchiverService(l, cfg, storageClient, beaconClient, m, registry) + return service.NewService(l, cfg, storageClient, beaconClient, m) } } diff --git a/archiver/metrics/metrics.go b/archiver/metrics/metrics.go index b05f9ab..af14c75 100644 --- a/archiver/metrics/metrics.go +++ b/archiver/metrics/metrics.go @@ -2,19 +2,21 @@ package metrics import ( "github.com/ethereum-optimism/optimism/op-service/metrics" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/prometheus/client_golang/prometheus" ) type BlockSource string var ( - metricsNamespace = "blob_archiver" + MetricsNamespace = "blob_archiver" BlockSourceBackfill BlockSource = "backfill" BlockSourceLive BlockSource = "live" ) type Metricer interface { + Registry() *prometheus.Registry RecordProcessedBlock(source BlockSource) RecordStoredBlobs(count int) } @@ -22,24 +24,31 @@ type Metricer interface { type metricsRecorder struct { blockProcessedCounter *prometheus.CounterVec blobsStored prometheus.Counter + registry *prometheus.Registry } -func NewMetrics(registry *prometheus.Registry) Metricer { +func NewMetrics() Metricer { + registry := opmetrics.NewRegistry() factory := metrics.With(registry) return &metricsRecorder{ + registry: registry, blockProcessedCounter: factory.NewCounterVec(prometheus.CounterOpts{ - Namespace: metricsNamespace, + Namespace: MetricsNamespace, Name: "blocks_processed", Help: "number of times processing loop has run", }, []string{"source"}), blobsStored: factory.NewCounter(prometheus.CounterOpts{ - Namespace: metricsNamespace, + Namespace: MetricsNamespace, Name: "blobs_stored", Help: "number of blobs stored", }), } } +func (m *metricsRecorder) Registry() *prometheus.Registry { + return m.registry +} + func (m *metricsRecorder) RecordStoredBlobs(count int) { m.blobsStored.Add(float64(count)) } diff --git a/archiver/service/api.go b/archiver/service/api.go new file mode 100644 index 0000000..aa059b8 --- /dev/null +++ b/archiver/service/api.go @@ -0,0 +1,45 @@ +package service + +import ( + "net/http" + "time" + + m "github.com/base-org/blob-archiver/archiver/metrics" + opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" + "github.com/ethereum/go-ethereum/log" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +const ( + serverTimeout = 60 * time.Second +) + +type API struct { + router *chi.Mux + logger log.Logger + metrics m.Metricer +} + +func NewAPI(metrics m.Metricer, logger log.Logger) *API { + result := &API{ + router: chi.NewRouter(), + logger: logger, + metrics: metrics, + } + + r := result.router + r.Use(middleware.Logger) + r.Use(middleware.Timeout(serverTimeout)) + r.Use(middleware.Recoverer) + r.Use(middleware.Heartbeat("/healthz")) + + recorder := opmetrics.NewPromHTTPRecorder(metrics.Registry(), m.MetricsNamespace) + r.Use(func(handler http.Handler) http.Handler { + return opmetrics.NewHTTPRecordingMiddleware(recorder, handler) + }) + + r.Get("/", http.NotFound) + + return result +} diff --git a/archiver/service/api_test.go b/archiver/service/api_test.go new file mode 100644 index 0000000..8f2af61 --- /dev/null +++ b/archiver/service/api_test.go @@ -0,0 +1,28 @@ +package service + +import ( + "net/http/httptest" + "testing" + + "github.com/base-org/blob-archiver/archiver/metrics" + "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum/go-ethereum/log" + "github.com/stretchr/testify/require" +) + +func setupAPI(t *testing.T) *API { + logger := testlog.Logger(t, log.LvlInfo) + m := metrics.NewMetrics() + return NewAPI(m, logger) +} + +func TestHealthHandler(t *testing.T) { + a := setupAPI(t) + + request := httptest.NewRequest("GET", "/healthz", nil) + response := httptest.NewRecorder() + + a.router.ServeHTTP(response, request) + + require.Equal(t, 200, response.Code) +} diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go index d6a2a02..1fe6295 100644 --- a/archiver/service/archiver.go +++ b/archiver/service/archiver.go @@ -16,7 +16,6 @@ import ( opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/prometheus/client_golang/prometheus" ) const backfillErrorRetryInterval = 5 * time.Second @@ -28,11 +27,10 @@ type BeaconClient interface { client.BeaconBlockHeadersProvider } -func NewArchiverService(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer, registry *prometheus.Registry) (*ArchiverService, error) { +func NewService(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer) (*ArchiverService, error) { return &ArchiverService{ log: l, cfg: cfg, - registry: registry, dataStoreClient: dataStoreClient, metrics: m, stopCh: make(chan struct{}), @@ -46,7 +44,6 @@ type ArchiverService struct { log log.Logger dataStoreClient storage.DataStore beaconClient BeaconClient - registry *prometheus.Registry metricsServer *httputil.HTTPServer cfg flags.ArchiverConfig metrics metrics.Metricer @@ -55,7 +52,7 @@ type ArchiverService struct { func (a *ArchiverService) Start(ctx context.Context) error { if a.cfg.MetricsConfig.Enabled { a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort) - srv, err := opmetrics.StartServer(a.registry, a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort) + srv, err := opmetrics.StartServer(a.metrics.Registry(), a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort) if err != nil { return err } diff --git a/archiver/service/archiver_test.go b/archiver/service/archiver_test.go index a7514d6..91860cd 100644 --- a/archiver/service/archiver_test.go +++ b/archiver/service/archiver_test.go @@ -14,19 +14,17 @@ import ( "github.com/ethereum-optimism/optimism/op-service/testlog" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" ) func setup(t *testing.T, beacon *beacontest.StubBeaconClient) (*ArchiverService, *storagetest.TestFileStorage) { l := testlog.Logger(t, log.LvlInfo) fs := storagetest.NewTestFileStorage(t, l) - registry := prometheus.NewRegistry() - m := metrics.NewMetrics(registry) - svc, err := NewArchiverService(l, flags.ArchiverConfig{ + m := metrics.NewMetrics() + svc, err := NewService(l, flags.ArchiverConfig{ PollInterval: 5 * time.Second, OriginBlock: blobtest.OriginBlock, - }, fs, beacon, m, registry) + }, fs, beacon, m) require.NoError(t, err) return svc, fs } From fb49b651894f7a493fc2b7f5e5ea9ef2cdc01c5e Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Mon, 12 Feb 2024 16:29:49 -0600 Subject: [PATCH 04/10] Add support for IAM access to S3 --- api/Makefile | 2 +- common/flags/config.go | 56 +++++++++++++++++++++++++++------------ common/flags/flags.go | 6 +++++ common/storage/s3.go | 9 ++++++- common/storage/s3_test.go | 11 ++++---- 5 files changed, 60 insertions(+), 24 deletions(-) diff --git a/api/Makefile b/api/Makefile index c4b1e87..e809280 100644 --- a/api/Makefile +++ b/api/Makefile @@ -10,4 +10,4 @@ test: .PHONY: \ blob-api \ clean \ - test \ No newline at end of file + test diff --git a/common/flags/config.go b/common/flags/config.go index 869092a..1972a2d 100644 --- a/common/flags/config.go +++ b/common/flags/config.go @@ -8,19 +8,25 @@ import ( ) type DataStorage string +type S3CredentialType string const ( - DataStorageUnknown DataStorage = "unknown" - DataStorageS3 DataStorage = "s3" - DataStorageFile DataStorage = "file" + DataStorageUnknown DataStorage = "unknown" + DataStorageS3 DataStorage = "s3" + DataStorageFile DataStorage = "file" + S3CredentialUnknown S3CredentialType = "unknown" + S3CredentialStatic S3CredentialType = "static" + S3CredentialIAM S3CredentialType = "iam" ) type S3Config struct { - Endpoint string - AccessKey string - SecretAccessKey string - UseHttps bool - Bucket string + Endpoint string + UseHttps bool + Bucket string + + S3CredentialType S3CredentialType + AccessKey string + SecretAccessKey string } func (c S3Config) check() error { @@ -28,12 +34,18 @@ func (c S3Config) check() error { return errors.New("s3 endpoint must be set") } - if c.AccessKey == "" { - return errors.New("s3 access key must be set") + if c.S3CredentialType == S3CredentialUnknown { + return errors.New("s3 credential type must be set") } - if c.SecretAccessKey == "" { - return errors.New("s3 secret access key must be set") + if c.S3CredentialType == S3CredentialStatic { + if c.AccessKey == "" { + return errors.New("s3 access key must be set") + } + + if c.SecretAccessKey == "" { + return errors.New("s3 secret access key must be set") + } } if c.Bucket == "" { @@ -85,12 +97,22 @@ func toDataStorage(s string) DataStorage { func readS3Config(ctx *cli.Context) S3Config { return S3Config{ - Endpoint: ctx.String(S3EndpointFlagName), - AccessKey: ctx.String(S3AccessKeyFlagName), - SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName), - UseHttps: ctx.Bool(S3EndpointHttpsFlagName), - Bucket: ctx.String(S3BucketFlagName), + Endpoint: ctx.String(S3EndpointFlagName), + AccessKey: ctx.String(S3AccessKeyFlagName), + SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName), + UseHttps: ctx.Bool(S3EndpointHttpsFlagName), + Bucket: ctx.String(S3BucketFlagName), + S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)), + } +} + +func toS3CredentialType(s string) S3CredentialType { + if s == string(S3CredentialStatic) { + return S3CredentialStatic + } else if s == string(S3CredentialIAM) { + return S3CredentialIAM } + return S3CredentialUnknown } func (c BeaconConfig) Check() error { diff --git a/common/flags/flags.go b/common/flags/flags.go index 86a09d4..1bd30b7 100644 --- a/common/flags/flags.go +++ b/common/flags/flags.go @@ -9,6 +9,7 @@ const ( BeaconHttpFlagName = "l1-beacon-http" BeaconHttpClientTimeoutFlagName = "l1-beacon-client-timeout" DataStoreFlagName = "data-store" + S3CredentialTypeFlagName = "s3-credential-type" S3EndpointFlagName = "s3-endpoint" S3EndpointHttpsFlagName = "s3-endpoint-https" S3AccessKeyFlagName = "s3-access-key" @@ -34,6 +35,11 @@ func CLIFlags(envPrefix string) []cli.Flag { }, // Optional Flags // S3 Data Store Flags + &cli.StringFlag{ + Name: S3CredentialTypeFlagName, + Usage: "The way to authenticate to S3, options are [iam, static]", + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_CREDENTIAL_TYPE"), + }, &cli.StringFlag{ Name: S3EndpointFlagName, Usage: "The URL for the S3 bucket (without the scheme http or https specified)", diff --git a/common/storage/s3.go b/common/storage/s3.go index c5e4bf7..8bfb112 100644 --- a/common/storage/s3.go +++ b/common/storage/s3.go @@ -19,8 +19,15 @@ type S3Storage struct { } func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) { + var c *credentials.Credentials + if cfg.S3CredentialType == flags.S3CredentialStatic { + c = credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, "") + } else { + c = credentials.NewIAM("") + } + client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, ""), + Creds: c, Secure: cfg.UseHttps, }) diff --git a/common/storage/s3_test.go b/common/storage/s3_test.go index 9daa894..3a233a2 100644 --- a/common/storage/s3_test.go +++ b/common/storage/s3_test.go @@ -24,11 +24,12 @@ func setupS3(t *testing.T) *S3Storage { l := testlog.Logger(t, log.LvlInfo) s3, err := NewS3Storage(flags.S3Config{ - Endpoint: "localhost:9000", - AccessKey: "admin", - SecretAccessKey: "password", - UseHttps: false, - Bucket: "blobs", + Endpoint: "localhost:9000", + AccessKey: "admin", + SecretAccessKey: "password", + UseHttps: false, + Bucket: "blobs", + S3CredentialType: flags.S3CredentialStatic, }, l) require.NoError(t, err) From d8c64a9aab092c8116f946a24540634bf1f40db1 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Wed, 14 Feb 2024 10:09:59 -0600 Subject: [PATCH 05/10] Fix typo --- api/service/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/service.go b/api/service/service.go index 84bc963..10c04f5 100644 --- a/api/service/service.go +++ b/api/service/service.go @@ -62,7 +62,7 @@ func (a *APIService) Stop(ctx context.Context) error { if a.stopped.Load() { return ErrAlreadyStopped } - a.log.Info("Stopping Archiver") + a.log.Info("Stopping API") a.stopped.Store(true) if a.apiServer != nil { From 6bd95f605e6a61af10b36fc42ada4dce6aadb8d9 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Wed, 14 Feb 2024 13:38:22 -0600 Subject: [PATCH 06/10] Run archiver api --- api/flags/flags.go | 2 +- archiver/cmd/main.go | 4 +++- archiver/flags/config.go | 6 ++++++ archiver/flags/flags.go | 10 ++++++++-- archiver/service/archiver.go | 12 +++++++++++- archiver/service/archiver_test.go | 3 ++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/api/flags/flags.go b/api/flags/flags.go index dd74bdc..aa5500a 100644 --- a/api/flags/flags.go +++ b/api/flags/flags.go @@ -12,7 +12,7 @@ const EnvVarPrefix = "BLOB_API" var ( ListenAddressFlag = &cli.StringFlag{ - Name: "api-list-address", + Name: "api-listen-address", Usage: "The address to list for new requests on", EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"), Value: "0.0.0.0:8000", diff --git a/archiver/cmd/main.go b/archiver/cmd/main.go index 4915e9d..00535f6 100644 --- a/archiver/cmd/main.go +++ b/archiver/cmd/main.go @@ -66,7 +66,9 @@ func Main() cliapp.LifecycleAction { return nil, err } + api := service.NewAPI(m, l) + l.Info("Initializing Archiver Service") - return service.NewService(l, cfg, storageClient, beaconClient, m) + return service.NewService(l, cfg, api, storageClient, beaconClient, m) } } diff --git a/archiver/flags/config.go b/archiver/flags/config.go index 2e5504a..d680e6e 100644 --- a/archiver/flags/config.go +++ b/archiver/flags/config.go @@ -18,6 +18,7 @@ type ArchiverConfig struct { StorageConfig common.StorageConfig PollInterval time.Duration OriginBlock geth.Hash + ListenAddr string } func (c ArchiverConfig) Check() error { @@ -37,6 +38,10 @@ func (c ArchiverConfig) Check() error { return fmt.Errorf("invalid origin block") } + if c.ListenAddr == "" { + return fmt.Errorf("archiver listen address must be set") + } + return nil } @@ -49,5 +54,6 @@ func ReadConfig(cliCtx *cli.Context) ArchiverConfig { StorageConfig: common.NewStorageConfig(cliCtx), PollInterval: pollInterval, OriginBlock: geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)), + ListenAddr: cliCtx.String(ArchiverListenAddrFlag.Name), } } diff --git a/archiver/flags/flags.go b/archiver/flags/flags.go index 7032cbc..4437ae2 100644 --- a/archiver/flags/flags.go +++ b/archiver/flags/flags.go @@ -19,10 +19,16 @@ var ( } ArchiverOriginBlock = &cli.StringFlag{ Name: "archiver-origin-block", - Usage: "The lastest block hash that the archiver will walk back to", + Usage: "The latest block hash that the archiver will walk back to", Required: true, EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "ORIGIN_BLOCK"), } + ArchiverListenAddrFlag = &cli.StringFlag{ + Name: "archiver-listen-address", + Usage: "The address to list for new requests on", + EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"), + Value: "0.0.0.0:8000", + } ) func init() { @@ -31,7 +37,7 @@ func init() { flags = append(flags, common.CLIFlags(EnvVarPrefix)...) flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...) flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...) - flags = append(flags, ArchiverPollIntervalFlag, ArchiverOriginBlock) + flags = append(flags, ArchiverPollIntervalFlag, ArchiverOriginBlock, ArchiverListenAddrFlag) Flags = flags } diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go index 1fe6295..0f17b5d 100644 --- a/archiver/service/archiver.go +++ b/archiver/service/archiver.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "fmt" "sync/atomic" "time" @@ -27,7 +28,7 @@ type BeaconClient interface { client.BeaconBlockHeadersProvider } -func NewService(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer) (*ArchiverService, error) { +func NewService(l log.Logger, cfg flags.ArchiverConfig, api *API, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer) (*ArchiverService, error) { return &ArchiverService{ log: l, cfg: cfg, @@ -35,6 +36,7 @@ func NewService(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage. metrics: m, stopCh: make(chan struct{}), beaconClient: client, + api: api, }, nil } @@ -47,6 +49,7 @@ type ArchiverService struct { metricsServer *httputil.HTTPServer cfg flags.ArchiverConfig metrics metrics.Metricer + api *API } func (a *ArchiverService) Start(ctx context.Context) error { @@ -61,6 +64,13 @@ func (a *ArchiverService) Start(ctx context.Context) error { a.metricsServer = srv } + srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.api.router) + if err != nil { + return fmt.Errorf("failed to start Archiver API server: %w", err) + } + + a.log.Info("Archiver API server started", "address", srv.Addr().String()) + currentBlob, _, err := a.persistBlobsForBlockToS3(ctx, "head") if err != nil { a.log.Error("failed to seed archiver with initial block", "err", err) diff --git a/archiver/service/archiver_test.go b/archiver/service/archiver_test.go index 91860cd..c334bd3 100644 --- a/archiver/service/archiver_test.go +++ b/archiver/service/archiver_test.go @@ -21,10 +21,11 @@ func setup(t *testing.T, beacon *beacontest.StubBeaconClient) (*ArchiverService, l := testlog.Logger(t, log.LvlInfo) fs := storagetest.NewTestFileStorage(t, l) m := metrics.NewMetrics() + svc, err := NewService(l, flags.ArchiverConfig{ PollInterval: 5 * time.Second, OriginBlock: blobtest.OriginBlock, - }, fs, beacon, m) + }, NewAPI(m, l), fs, beacon, m) require.NoError(t, err) return svc, fs } From 7927f72db98b4859451564ab94c89dd3901adb07 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Wed, 14 Feb 2024 16:57:22 -0600 Subject: [PATCH 07/10] Add godoc / notice about trusting beacon client --- README.md | 5 +++++ api/service/api.go | 2 ++ archiver/service/api.go | 1 + archiver/service/archiver.go | 18 ++++++++++++++++++ common/beacon/client.go | 2 ++ common/storage/storage.go | 24 +++++++++++++++++++++++- 6 files changed, 51 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c65a36..74b2a0e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ There are currently two supported storage options: You can control which storage backend is used by setting the `BLOB_API_DATA_STORE` and `BLOB_ARCHIVER_DATA_STORE` to either `disk` or `s3`. +### Data Validity +Currently, the archiver and api do not validate the beacon node's data. Therefore, it's important to either trust the +Beacon node, or validate the data in the client. There is an open [issue](https://github.com/base-org/blob-archiver/issues/4) +to add data validation to the archiver and api. + ### Development The `Makefile` contains a number of commands for development: diff --git a/api/service/api.go b/api/service/api.go index eddfc97..6b1ecd9 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -150,6 +150,8 @@ func (a *API) toBeaconBlockHash(id string) (common.Hash, *httpError) { } } +// blobSidecarHandler implements the /eth/v1/beacon/blob_sidecars/{id} endpoint, using the underlying DataStoreReader +// to fetch blobs instead of the beacon node. This allows clients to fetch expired blobs. func (a *API) blobSidecarHandler(w http.ResponseWriter, r *http.Request) { param := chi.URLParam(r, "id") beaconBlockHash, err := a.toBeaconBlockHash(param) diff --git a/archiver/service/api.go b/archiver/service/api.go index aa059b8..59e64d9 100644 --- a/archiver/service/api.go +++ b/archiver/service/api.go @@ -21,6 +21,7 @@ type API struct { metrics m.Metricer } +// NewAPI creates a new Archiver API instance. This API exposes an admin interface to control the archiver. func NewAPI(metrics m.Metricer, logger log.Logger) *API { result := &API{ router: chi.NewRouter(), diff --git a/archiver/service/archiver.go b/archiver/service/archiver.go index 0f17b5d..efbc35e 100644 --- a/archiver/service/archiver.go +++ b/archiver/service/archiver.go @@ -52,6 +52,10 @@ type ArchiverService struct { api *API } +// Start starts the archiver service. It begins polling the beacon node for the latest blocks and persisting blobs for +// them. Concurrently it'll also begin a backfill process (see backfillBlobs) to store all blobs from the current head +// to the previously stored blocks. This ensures that during restarts or outages of an archiver, any gaps will be +// filled in. func (a *ArchiverService) Start(ctx context.Context) error { if a.cfg.MetricsConfig.Enabled { a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort) @@ -82,6 +86,11 @@ func (a *ArchiverService) Start(ctx context.Context) error { return a.trackLatestBlocks(ctx) } +// persistBlobsForBlockToS3 fetches the blobs for a given block and persists them to S3. It returns the block header +// and a boolean indicating whether the blobs already existed in S3 and any errors that occur. +// If the blobs are already stored, it will not overwrite the data. Currently, the archiver does not +// perform any validation of the blobs, it assumes a trusted beacon node. See: +// https://github.com/base-org/blob-archiver/issues/4. func (a *ArchiverService) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier string) (*v1.BeaconBlockHeader, bool, error) { currentHeader, err := a.beaconClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{ Block: blockIdentifier, @@ -121,6 +130,7 @@ func (a *ArchiverService) persistBlobsForBlockToS3(ctx context.Context, blockIde BlobSidecars: storage.BlobSidecars{Data: blobSidecars.Data}, } + // The blob that is being written has not been validated. It is assumed that the beacon node is trusted. err = a.dataStoreClient.Write(ctx, blobData) if err != nil { @@ -133,6 +143,7 @@ func (a *ArchiverService) persistBlobsForBlockToS3(ctx context.Context, blockIde return currentHeader.Data, false, nil } +// Stops the archiver service. func (a *ArchiverService) Stop(ctx context.Context) error { if a.stopped.Load() { return ErrAlreadyStopped @@ -155,6 +166,9 @@ func (a *ArchiverService) Stopped() bool { return a.stopped.Load() } +// backfillBlobs will persist all blobs from the provided beacon block header, to either the last block that was persisted +// to the archivers storage or the origin block in the configuration. This is used to ensure that any gaps can be filled. +// If an error is encountered persisting a block, it will retry after waiting for a period of time. func (a *ArchiverService) backfillBlobs(ctx context.Context, latest *v1.BeaconBlockHeader) { current, alreadyExists, err := latest, false, error(nil) @@ -182,6 +196,7 @@ func (a *ArchiverService) backfillBlobs(ctx context.Context, latest *v1.BeaconBl a.log.Info("backfill complete", "endHash", current.Root.String(), "startHash", latest.Root.String()) } +// trackLatestBlocks will poll the beacon node for the latest blocks and persist blobs for them. func (a *ArchiverService) trackLatestBlocks(ctx context.Context) error { t := time.NewTicker(a.cfg.PollInterval) defer t.Stop() @@ -198,6 +213,9 @@ func (a *ArchiverService) trackLatestBlocks(ctx context.Context) error { } } +// processBlocksUntilKnownBlock will fetch and persist blobs for blocks until it finds a block that has been stored before. +// In the case of a reorg, it will fetch the new head and then walk back the chain, storing all blobs until it finds a +// known block -- that already exists in the archivers' storage. func (a *ArchiverService) processBlocksUntilKnownBlock(ctx context.Context) { a.log.Debug("refreshing live data") diff --git a/common/beacon/client.go b/common/beacon/client.go index 0715ca7..1bd4937 100644 --- a/common/beacon/client.go +++ b/common/beacon/client.go @@ -8,11 +8,13 @@ import ( "github.com/base-org/blob-archiver/common/flags" ) +// Client is an interface that wraps the go-eth-2 interfaces that the blob archiver and api require. type Client interface { client.BeaconBlockHeadersProvider client.BlobSidecarsProvider } +// NewBeaconClient returns a new HTTP beacon client. func NewBeaconClient(ctx context.Context, cfg flags.BeaconConfig) (Client, error) { cctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/common/storage/storage.go b/common/storage/storage.go index 6db3f0a..e889a18 100644 --- a/common/storage/storage.go +++ b/common/storage/storage.go @@ -15,8 +15,11 @@ const ( ) var ( + // ErrNotFound is returned when a blob is not found in the storage. ErrNotFound = errors.New("blob not found") - ErrStorage = errors.New("error accessing storage") + // ErrStorage is returned when there is an error accessing the storage. + ErrStorage = errors.New("error accessing storage") + // ErrEncoding is returned when there is an error in blob encoding or decoding. ErrEncoding = errors.New("error encoding/decoding blob") ) @@ -28,6 +31,8 @@ type BlobSidecars struct { Data []*deneb.BlobSidecar `json:"data"` } +// MarshalSSZ marshals the blob sidecars into SSZ. As the blob sidecars are a single list of fixed size elements, we can +// simply concatenate the marshaled sidecars together. func (b *BlobSidecars) MarshalSSZ() ([]byte, error) { result := make([]byte, b.SizeSSZ()) @@ -55,15 +60,32 @@ type BlobData struct { BlobSidecars BlobSidecars `json:"blob_sidecars"` } +// DataStoreReader is the interface for reading from a data store. type DataStoreReader interface { + // Exists returns true if the given blob hash exists in the data store, false otherwise. + // It should return one of the following: + // - nil: the existence check was successful. In this case the boolean should also be set correctly. + // - ErrStorage: there was an error accessing the data store. Exists(ctx context.Context, hash common.Hash) (bool, error) + // Read reads the blob data for the given beacon block hash from the data store. + // It should return one of the following: + // - nil: reading the blob was successful. The blob data is also returned. + // - ErrNotFound: the blob data was not found in the data store. + // - ErrStorage: there was an error accessing the data store. + // - ErrEncoding: there was an error decoding the blob data. Read(ctx context.Context, hash common.Hash) (BlobData, error) } +// DataStoreWriter is the interface for writing to a data store. type DataStoreWriter interface { + // Write writes the given blob data to the data store. It should return one of the following errors: + // - nil: writing the blob was successful. + // - ErrStorage: there was an error accessing the data store. + // - ErrEncoding: there was an error encoding the blob data. Write(ctx context.Context, data BlobData) error } +// DataStore is the interface for a data store that can be both written to and read from. type DataStore interface { DataStoreReader DataStoreWriter From 043afb1a9779dfc4dd2edb0b7bfc8d1559d010bc Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Wed, 14 Feb 2024 19:09:44 -0600 Subject: [PATCH 08/10] Code review feedback --- README.md | 4 ++-- api/cmd/main.go | 4 ++-- api/flags/config.go | 4 ++-- api/flags/flags.go | 12 ++++-------- api/metrics/metrics.go | 10 ++++++---- api/service/api.go | 9 +++------ archiver/flags/flags.go | 12 ++++-------- 7 files changed, 23 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 74b2a0e..9df8d98 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Blob Archiver -The Blob Archiver is a service to archive and query all historical blobs from the beacon chain. It consistens of two -components: +The Blob Archiver is a service to archive and allow querying of all historical blobs from the beacon chain. It consists +of two components: * **Archiver** - Tracks the beacon chain and writes blobs to a storage backend * **API** - Implements the blob sidecars [API](https://ethereum.github.io/beacon-APIs/#/Beacon/getBlobSidecars), which diff --git a/api/cmd/main.go b/api/cmd/main.go index 70c15be..32f4971 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -57,12 +57,12 @@ func Main() cliapp.LifecycleAction { storageClient, err := storage.NewStorage(cfg.StorageConfig, l) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize storage: %w", err) } beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize beacon client: %w", err) } l.Info("Initializing API Service") diff --git a/api/flags/config.go b/api/flags/config.go index 1d615cc..9a9e266 100644 --- a/api/flags/config.go +++ b/api/flags/config.go @@ -20,11 +20,11 @@ type APIConfig struct { func (c APIConfig) Check() error { if err := c.StorageConfig.Check(); err != nil { - return err + return fmt.Errorf("storage config check failed: %w", err) } if err := c.BeaconConfig.Check(); err != nil { - return err + return fmt.Errorf("beacon config check failed: %w", err) } if c.ListenAddr == "" { diff --git a/api/flags/flags.go b/api/flags/flags.go index aa5500a..becc298 100644 --- a/api/flags/flags.go +++ b/api/flags/flags.go @@ -20,14 +20,10 @@ var ( ) func init() { - var flags []cli.Flag - - flags = append(flags, common.CLIFlags(EnvVarPrefix)...) - flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...) - flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...) - flags = append(flags, ListenAddressFlag) - - Flags = flags + Flags = append(Flags, common.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, ListenAddressFlag) } // Flags contains the list of configuration options available to the binary. diff --git a/api/metrics/metrics.go b/api/metrics/metrics.go index aa26d17..92e5981 100644 --- a/api/metrics/metrics.go +++ b/api/metrics/metrics.go @@ -22,8 +22,10 @@ type Metricer interface { } type metricsRecorder struct { - inputType *prometheus.CounterVec - registry *prometheus.Registry + // blockIdType records the type of block id used to request a block. This could be a hash (BlockIdTypeHash), or a + // beacon block identifier (BlockIdTypeBeacon). + blockIdType *prometheus.CounterVec + registry *prometheus.Registry } func NewMetrics() Metricer { @@ -31,7 +33,7 @@ func NewMetrics() Metricer { factory := metrics.With(registry) return &metricsRecorder{ registry: registry, - inputType: factory.NewCounterVec(prometheus.CounterOpts{ + blockIdType: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: MetricsNamespace, Name: "block_id_type", Help: "The type of block id used to request a block", @@ -40,7 +42,7 @@ func NewMetrics() Metricer { } func (m *metricsRecorder) RecordBlockIdType(t BlockIdType) { - m.inputType.WithLabelValues(string(t)).Inc() + m.blockIdType.WithLabelValues(string(t)).Inc() } func (m *metricsRecorder) Registry() *prometheus.Registry { diff --git a/api/service/api.go b/api/service/api.go index 6b1ecd9..c1cf876 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -133,11 +133,8 @@ func (a *API) toBeaconBlockHash(id string) (common.Hash, *httpError) { if err != nil { var apiErr *api.Error - if errors.As(err, &apiErr) { - switch apiErr.StatusCode { - case 404: - return common.Hash{}, errUnknownBlock - } + if errors.As(err, &apiErr) && apiErr.StatusCode == 404 { + return common.Hash{}, errUnknownBlock } return common.Hash{}, errServerError @@ -208,7 +205,7 @@ func (a *API) blobSidecarHandler(w http.ResponseWriter, r *http.Request) { } // filterBlobs filters the blobs based on the indices query provided. -// If no indices or invalid indices are provided, the original blobs are returned. +// If no indices are provided, all blobs are returned. If invalid indices are provided, an error is returned. func filterBlobs(blobs []*deneb.BlobSidecar, indices string) ([]*deneb.BlobSidecar, *httpError) { if indices == "" { return blobs, nil diff --git a/archiver/flags/flags.go b/archiver/flags/flags.go index 4437ae2..9b7af1e 100644 --- a/archiver/flags/flags.go +++ b/archiver/flags/flags.go @@ -32,14 +32,10 @@ var ( ) func init() { - var flags []cli.Flag - - flags = append(flags, common.CLIFlags(EnvVarPrefix)...) - flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...) - flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...) - flags = append(flags, ArchiverPollIntervalFlag, ArchiverOriginBlock, ArchiverListenAddrFlag) - - Flags = flags + Flags = append(Flags, common.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...) + Flags = append(Flags, ArchiverPollIntervalFlag, ArchiverOriginBlock, ArchiverListenAddrFlag) } // Flags contains the list of configuration options available to the binary. From 87a76350843b6f0a5d9a4b0aca70abb9fe928a30 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Wed, 14 Feb 2024 19:56:47 -0600 Subject: [PATCH 09/10] Code review feedback (2) --- common/beacon/client.go | 2 +- common/flags/config.go | 9 +++++---- common/storage/file.go | 4 ++-- common/storage/file_test.go | 2 +- common/storage/s3.go | 4 ++-- common/storage/storage.go | 8 ++++---- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/common/beacon/client.go b/common/beacon/client.go index 1bd4937..229026c 100644 --- a/common/beacon/client.go +++ b/common/beacon/client.go @@ -19,7 +19,7 @@ func NewBeaconClient(ctx context.Context, cfg flags.BeaconConfig) (Client, error cctx, cancel := context.WithCancel(ctx) defer cancel() - c, err := http.New(cctx, http.WithAddress(cfg.BeaconUrl), http.WithTimeout(cfg.BeaconClientTimeout)) + c, err := http.New(cctx, http.WithAddress(cfg.BeaconURL), http.WithTimeout(cfg.BeaconClientTimeout)) if err != nil { return nil, err } diff --git a/common/flags/config.go b/common/flags/config.go index 1972a2d..1e6dfb3 100644 --- a/common/flags/config.go +++ b/common/flags/config.go @@ -2,6 +2,7 @@ package flags import ( "errors" + "fmt" "time" "github.com/urfave/cli/v2" @@ -56,7 +57,7 @@ func (c S3Config) check() error { } type BeaconConfig struct { - BeaconUrl string + BeaconURL string BeaconClientTimeout time.Duration } @@ -70,7 +71,7 @@ func NewBeaconConfig(cliCtx *cli.Context) BeaconConfig { timeout, _ := time.ParseDuration(cliCtx.String(BeaconHttpClientTimeoutFlagName)) return BeaconConfig{ - BeaconUrl: cliCtx.String(BeaconHttpFlagName), + BeaconURL: cliCtx.String(BeaconHttpFlagName), BeaconClientTimeout: timeout, } } @@ -116,7 +117,7 @@ func toS3CredentialType(s string) S3CredentialType { } func (c BeaconConfig) Check() error { - if c.BeaconUrl == "" { + if c.BeaconURL == "" { return errors.New("beacon url must be set") } @@ -134,7 +135,7 @@ func (c StorageConfig) Check() error { if c.DataStorageType == DataStorageS3 { if err := c.S3Config.check(); err != nil { - return err + return fmt.Errorf("s3 config check failed: %w", err) } } else if c.DataStorageType == DataStorageFile && c.FileStorageDirectory == "" { return errors.New("file storage directory must be set") diff --git a/common/storage/file.go b/common/storage/file.go index 990b8cf..f143f64 100644 --- a/common/storage/file.go +++ b/common/storage/file.go @@ -46,7 +46,7 @@ func (s *FileStorage) Read(_ context.Context, hash common.Hash) (BlobData, error err = json.Unmarshal(data, &result) if err != nil { s.log.Warn("error decoding blob", "err", err, "hash", hash.String()) - return BlobData{}, ErrEncoding + return BlobData{}, ErrMarshaling } return result, nil } @@ -55,7 +55,7 @@ func (s *FileStorage) Write(_ context.Context, data BlobData) error { b, err := json.Marshal(data) if err != nil { s.log.Warn("error encoding blob", "err", err) - return ErrEncoding + return ErrMarshaling } err = os.WriteFile(s.fileName(data.Header.BeaconBlockHash), b, 0644) if err != nil { diff --git a/common/storage/file_test.go b/common/storage/file_test.go index 3f3ed03..b987099 100644 --- a/common/storage/file_test.go +++ b/common/storage/file_test.go @@ -111,5 +111,5 @@ func TestReadInvalidData(t *testing.T) { _, err = fs.Read(context.Background(), id) require.Error(t, err) - require.True(t, errors.Is(err, ErrEncoding)) + require.True(t, errors.Is(err, ErrMarshaling)) } diff --git a/common/storage/s3.go b/common/storage/s3.go index 8bfb112..91981a5 100644 --- a/common/storage/s3.go +++ b/common/storage/s3.go @@ -79,7 +79,7 @@ func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error err = json.NewDecoder(res).Decode(&data) if err != nil { s.log.Warn("error decoding blob", "hash", hash.String(), "err", err) - return BlobData{}, ErrEncoding + return BlobData{}, ErrMarshaling } return data, nil @@ -89,7 +89,7 @@ func (s *S3Storage) Write(ctx context.Context, data BlobData) error { b, err := json.Marshal(data) if err != nil { s.log.Warn("error encoding blob", "err", err) - return ErrEncoding + return ErrMarshaling } reader := bytes.NewReader(b) diff --git a/common/storage/storage.go b/common/storage/storage.go index e889a18..cdad499 100644 --- a/common/storage/storage.go +++ b/common/storage/storage.go @@ -19,8 +19,8 @@ var ( ErrNotFound = errors.New("blob not found") // ErrStorage is returned when there is an error accessing the storage. ErrStorage = errors.New("error accessing storage") - // ErrEncoding is returned when there is an error in blob encoding or decoding. - ErrEncoding = errors.New("error encoding/decoding blob") + // ErrMarshaling is returned when there is an error in (un)marshaling the blob + ErrMarshaling = errors.New("error encoding/decoding blob") ) type Header struct { @@ -72,7 +72,7 @@ type DataStoreReader interface { // - nil: reading the blob was successful. The blob data is also returned. // - ErrNotFound: the blob data was not found in the data store. // - ErrStorage: there was an error accessing the data store. - // - ErrEncoding: there was an error decoding the blob data. + // - ErrMarshaling: there was an error decoding the blob data. Read(ctx context.Context, hash common.Hash) (BlobData, error) } @@ -81,7 +81,7 @@ type DataStoreWriter interface { // Write writes the given blob data to the data store. It should return one of the following errors: // - nil: writing the blob was successful. // - ErrStorage: there was an error accessing the data store. - // - ErrEncoding: there was an error encoding the blob data. + // - ErrMarshaling: there was an error encoding the blob data. Write(ctx context.Context, data BlobData) error } From 3714928d39125f3dac8217c23c695dff4c1146da Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Thu, 15 Feb 2024 13:34:54 -0600 Subject: [PATCH 10/10] Error on out of range indices --- api/service/api.go | 12 ++++++++++++ api/service/api_test.go | 28 ++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/api/service/api.go b/api/service/api.go index c1cf876..f89e5ba 100644 --- a/api/service/api.go +++ b/api/service/api.go @@ -68,6 +68,13 @@ func newIndicesError(input string) *httpError { } } +func newOutOfRangeError(input uint64, blobCount int) *httpError { + return &httpError{ + Code: http.StatusBadRequest, + Message: fmt.Sprintf("invalid index: %d block contains %d blobs", input, blobCount), + } +} + type API struct { dataStoreClient storage.DataStoreReader beaconClient client.BeaconBlockHeadersProvider @@ -222,6 +229,11 @@ func filterBlobs(blobs []*deneb.BlobSidecar, indices string) ([]*deneb.BlobSidec if err != nil { return nil, newIndicesError(index) } + + if parsedInt >= uint64(len(blobs)) { + return nil, newOutOfRangeError(parsedInt, len(blobs)) + } + blobIndex := deneb.BlobIndex(parsedInt) indicesMap[blobIndex] = struct{}{} } diff --git a/api/service/api_test.go b/api/service/api_test.go index e597e9c..afaa6be 100644 --- a/api/service/api_test.go +++ b/api/service/api_test.go @@ -171,12 +171,28 @@ func TestAPIService(t *testing.T) { }, }, { - name: "indices out of bounds returns empty array", - path: "/eth/v1/beacon/blob_sidecars/1234?indices=3", - status: 200, - expected: &storage.BlobSidecars{ - Data: []*deneb.BlobSidecar{}, - }, + name: "only index out of bounds returns empty array", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=3", + status: 400, + errMessage: "invalid index: 3 block contains 2 blobs", + }, + { + name: "any index out of bounds returns error", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=1,10", + status: 400, + errMessage: "invalid index: 10 block contains 2 blobs", + }, + { + name: "only index out of bounds (boundary condition) returns error", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=2", + status: 400, + errMessage: "invalid index: 2 block contains 2 blobs", + }, + { + name: "negative index returns error", + path: "/eth/v1/beacon/blob_sidecars/1234?indices=-2", + status: 400, + errMessage: "invalid index input: -2", }, { name: "no 0x on hash",