Initial implementation of blob archiver service

Co-authored-by: Qi Wu <[email protected]>

Author: danyalprout

.env.template

@@ -0,0 +1,22 @@
+# To get started, copy this file to .env and set your beacon http endpoint
+
+BLOB_ARCHIVER_L1_BEACON_HTTP=<unset>
+BLOB_ARCHIVER_DATA_STORE=s3
+BLOB_ARCHIVER_S3_ENDPOINT=172.17.0.1:9000
+BLOB_ARCHIVER_S3_ACCESS_KEY=admin
+BLOB_ARCHIVER_S3_SECRET_ACCESS_KEY=password
+BLOB_ARCHIVER_S3_ENDPOINT_HTTPS=false
+BLOB_ARCHIVER_S3_BUCKET=blobs
+BLOB_ARCHIVER_METRICS_ENABLED=true
+BLOB_ARCHIVER_METRICS_PORT=7300
+BLOB_ARCHIVER_ORIGIN_BLOCK=0x0
+
+BLOB_API_L1_BEACON_HTTP=<unset>
+BLOB_API_DATA_STORE=s3
+BLOB_API_S3_ENDPOINT=172.17.0.1:9000
+BLOB_API_S3_ACCESS_KEY=admin
+BLOB_API_S3_SECRET_ACCESS_KEY=password
+BLOB_API_S3_ENDPOINT_HTTPS=false
+BLOB_API_S3_BUCKET=blobs
+BLOB_API_METRICS_ENABLED=true
+BLOB_API_METRICS_PORT=7301

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+.DS_Store
+.swp
+.env
+api/bin
+archiver/bin

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.21.6-alpine3.19 as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers jq bash
+
+WORKDIR /app
+
+COPY ./go.mod ./go.sum /app/
+
+RUN go mod download
+
+COPY . /app
+
+RUN make build
+
+FROM alpine:3.19
+
+COPY --from=builder /app/archiver/bin/blob-archiver /usr/local/bin/blob-archiver
+COPY --from=builder /app/api/bin/blob-api /usr/local/bin/blob-api

Makefile

@@ -0,0 +1,35 @@
+build:
+	make -C ./archiver blob-archiver
+	make -C ./api blob-api
+.PHONY: build
+
+build-docker:
+	docker-compose build
+.PHONY: build-docker
+
+clean:
+	make -C ./archiver clean
+	make -C ./api clean
+.PHONY: clean
+
+test:
+	make -C ./archiver test
+	make -C ./api test
+.PHONY: test
+
+integration:
+	docker-compose down
+	docker-compose up -d minio create-buckets
+	RUN_INTEGRATION_TESTS=true go test -v ./...
+.PHONY: integration
+
+fmt:
+	gofmt -s -w .
+.PHONY: fmt
+
+check: fmt clean build build-docker lint test integration
+.PHONY: check
+
+lint:
+	golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
+.PHONY: lint

README.md

@@ -1 +1,53 @@
 # Blob Archiver
+The Blob Archiver is a service to archive and query all historical blobs from the beacon chain. It consistens of two
+components:
+
+* **Archiver** - Tracks the beacon chain and writes blobs to a storage backend
+* **API** - Implements the blob sidecars [API](https://ethereum.github.io/beacon-APIs/#/Beacon/getBlobSidecars), which 
+allows clients to retrieve blobs from the storage backend
+
+### Storage
+There are currently two supported storage options:
+
+* On-disk storage - Blobs are written to disk in a directory
+* S3 storage - Blobs are written to an S3 bucket
+
+You can control which storage backend is used by setting the `BLOB_API_DATA_STORE` and `BLOB_ARCHIVER_DATA_STORE` to 
+either `disk` or `s3`.
+
+### Development
+The `Makefile` contains a number of commands for development:
+
+```sh
+# Run the tests
+make test
+# Run the integration tests (will start a local S3 bucket)
+make integration 
+
+# Lint the project
+make lint
+
+# Build the project
+make build
+
+# Check all tests, formatting, building
+make check
+```
+
+#### Run Locally
+To run the project locally, you should first copy `.env.template` to `.env` and then modify the environment variables
+to your beacon client and storage backend of choice. Then you can run the project with:
+
+```sh
+docker-compose up
+```
+
+You can see a full list of configuration options by running:
+```sh
+# API
+go run api/cmd/main.go
+
+# Archiver
+go run archiver/cmd/main.go
+
+```

api/Makefile

@@ -0,0 +1,13 @@
+blob-api:
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-api ./cmd/main.go
+
+clean:
+	rm -f bin/blob-api
+
+test:
+	go test -v -race ./...
+
+.PHONY: \
+	blob-api \
+	clean \
+	test

api/cmd/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/base-org/blob-archiver/api/flags"
+	"github.com/base-org/blob-archiver/api/metrics"
+	"github.com/base-org/blob-archiver/api/service"
+	"github.com/base-org/blob-archiver/common/beacon"
+	"github.com/base-org/blob-archiver/common/storage"
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/ethereum-optimism/optimism/op-service/cliapp"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	Version   = "v0.0.1"
+	GitCommit = ""
+	GitDate   = ""
+)
+
+func main() {
+	oplog.SetupDefaults()
+
+	app := cli.NewApp()
+	app.Flags = cliapp.ProtectFlags(flags.Flags)
+	app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "")
+	app.Name = "blob-api"
+	app.Usage = "API service for Ethereum blobs"
+	app.Description = "Service for fetching blob sidecars from a datastore"
+	app.Action = cliapp.LifecycleCmd(Main())
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Crit("Application failed", "message", err)
+	}
+}
+
+// Main is the entrypoint into the API.
+// This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed API Server.
+func Main() cliapp.LifecycleAction {
+	return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) {
+		cfg := flags.ReadConfig(cliCtx)
+		if err := cfg.Check(); err != nil {
+			return nil, fmt.Errorf("config check failed: %w", err)
+		}
+
+		l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig)
+		oplog.SetGlobalLogHandler(l.GetHandler())
+		opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l)
+
+		registry := opmetrics.NewRegistry()
+		m := metrics.NewMetrics(registry)
+
+		storageClient, err := storage.NewStorage(cfg.StorageConfig, l)
+		if err != nil {
+			return nil, err
+		}
+
+		beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		l.Info("Initializing API Service")
+		return service.NewAPIService(l, storageClient, beaconClient, cfg, registry, m), nil
+	}
+}

api/flags/config.go

@@ -0,0 +1,45 @@
+package flags
+
+import (
+	"fmt"
+
+	common "github.com/base-org/blob-archiver/common/flags"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/urfave/cli/v2"
+)
+
+type APIConfig struct {
+	LogConfig     oplog.CLIConfig
+	MetricsConfig opmetrics.CLIConfig
+	BeaconConfig  common.BeaconConfig
+	StorageConfig common.StorageConfig
+
+	ListenAddr string
+}
+
+func (c APIConfig) Check() error {
+	if err := c.StorageConfig.Check(); err != nil {
+		return err
+	}
+
+	if err := c.BeaconConfig.Check(); err != nil {
+		return err
+	}
+
+	if c.ListenAddr == "" {
+		return fmt.Errorf("listen address must be set")
+	}
+
+	return nil
+}
+
+func ReadConfig(cliCtx *cli.Context) APIConfig {
+	return APIConfig{
+		LogConfig:     oplog.ReadCLIConfig(cliCtx),
+		MetricsConfig: opmetrics.ReadCLIConfig(cliCtx),
+		BeaconConfig:  common.NewBeaconConfig(cliCtx),
+		StorageConfig: common.NewStorageConfig(cliCtx),
+		ListenAddr:    cliCtx.String(ListenAddressFlag.Name),
+	}
+}

api/flags/flags.go

@@ -0,0 +1,34 @@
+package flags
+
+import (
+	common "github.com/base-org/blob-archiver/common/flags"
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/urfave/cli/v2"
+)
+
+const EnvVarPrefix = "BLOB_API"
+
+var (
+	ListenAddressFlag = &cli.StringFlag{
+		Name:    "api-list-address",
+		Usage:   "The address to list for new requests on",
+		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"),
+		Value:   "0.0.0.0:8000",
+	}
+)
+
+func init() {
+	var flags []cli.Flag
+
+	flags = append(flags, common.CLIFlags(EnvVarPrefix)...)
+	flags = append(flags, opmetrics.CLIFlags(EnvVarPrefix)...)
+	flags = append(flags, oplog.CLIFlags(EnvVarPrefix)...)
+	flags = append(flags, ListenAddressFlag)
+
+	Flags = flags
+}
+
+// Flags contains the list of configuration options available to the binary.
+var Flags []cli.Flag

api/metrics/metrics.go

@@ -0,0 +1,39 @@
+package metrics
+
+import (
+	"github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type BlockIdType string
+
+var (
+	MetricsNamespace = "blob_api"
+
+	BlockIdTypeHash    BlockIdType = "hash"
+	BlockIdTypeBeacon  BlockIdType = "beacon"
+	BlockIdTypeInvalid BlockIdType = "invalid"
+)
+
+type Metricer interface {
+	RecordBlockIdType(t BlockIdType)
+}
+
+type metricsRecorder struct {
+	inputType *prometheus.CounterVec
+}
+
+func NewMetrics(registry *prometheus.Registry) Metricer {
+	factory := metrics.With(registry)
+	return &metricsRecorder{
+		inputType: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Name:      "block_id_type",
+			Help:      "The type of block id used to request a block",
+		}, []string{"type"}),
+	}
+}
+
+func (m *metricsRecorder) RecordBlockIdType(t BlockIdType) {
+	m.inputType.WithLabelValues(string(t)).Inc()
+}