fix(server): own the vuls2 db for the process instead of per request

detector/vuls2/db.go

@@ -1,13 +1,15 @@
 package vuls2
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
 	bolt "go.etcd.io/bbolt"
 	"golang.org/x/xerrors"
+	"oras.land/oras-go/v2/registry/remote"
 
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/logging"
@@ -24,7 +26,18 @@ var (
 	}()
 )
 
-func newDBConfig(vuls2Conf config.Vuls2Conf, noProgress bool) (*session.Config, error) {
+// newDBConfig downloads or refreshes the db if due, validates its schema
+// version, and returns the session config to open it with.
+//
+// withCache decides whether the returned session carries a read cache. Any
+// session that answers a detection wants one: without it, enrichment re-reads
+// and re-unmarshals the same advisory and vulnerability records once per root
+// that references them, ~60x slower on a 4873-CVE result. The cache is
+// unbounded (a sync.Map that never evicts), so it is only safe on a session
+// whose lifetime is bounded — one server's detection run, or one request. A
+// session meant to outlive those must pass false, which is why SharedDB opens
+// the db without one when it is only checking that the db is usable.
+func newDBConfig(vuls2Conf config.Vuls2Conf, noProgress, withCache bool) (*session.Config, error) {
 	willDownload, err := shouldDownload(vuls2Conf, time.Now())
 	if err != nil {
 		return nil, xerrors.Errorf("Failed to check whether to download vuls2 db. err: %w", err)
@@ -70,10 +83,62 @@ func newDBConfig(vuls2Conf config.Vuls2Conf, noProgress bool) (*session.Config,
 		Type:      "boltdb",
 		Path:      vuls2Conf.Path,
 		Options:   session.StorageOptions{BoltDB: &bolt.Options{ReadOnly: true}},
-		WithCache: true,
+		WithCache: withCache,
 	}, nil
 }
 
+// hasNewerRemote reports whether the repository holds a db other than the one on
+// disk, comparing the manifest digest that fetch recorded in the local db's
+// metadata against the digest the repository's reference resolves to now.
+//
+// shouldDownload can only tell that the local db is old enough to be worth
+// checking: it goes by timestamps, and the nightly db's LastModified is the
+// night it was built, so a db past the staleness window looks due on every
+// check until something replaces it. Downloading on that alone re-fetches
+// gigabytes on every check even when the tag has not moved. Resolving the
+// manifest costs one request, so it is worth asking before spending a download.
+func hasNewerRemote(ctx context.Context, vuls2Conf config.Vuls2Conf) (bool, error) {
+	sesh, err := (&session.Config{
+		Type:    "boltdb",
+		Path:    vuls2Conf.Path,
+		Options: session.StorageOptions{BoltDB: &bolt.Options{ReadOnly: true}},
+	}).New()
+	if err != nil {
+		return false, xerrors.Errorf("Failed to new vuls2 db connection. path: %s, err: %w", vuls2Conf.Path, err)
+	}
+
+	if err := sesh.Storage().Open(); err != nil {
+		return false, xerrors.Errorf("Failed to open vuls2 db. path: %s, err: %w", vuls2Conf.Path, err)
+	}
+	defer sesh.Storage().Close()
+
+	metadata, err := sesh.Storage().GetMetadata()
+	if err != nil {
+		return false, xerrors.Errorf("Failed to get vuls2 db metadata. path: %s, err: %w", vuls2Conf.Path, err)
+	}
+	if metadata == nil || metadata.Digest == nil {
+		// A db that was built locally rather than fetched carries no digest, so
+		// there is nothing to compare it by and the repository's db counts as
+		// the newer one.
+		return true, nil
+	}
+
+	repo, err := remote.NewRepository(vuls2Conf.Repository)
+	if err != nil {
+		return false, xerrors.Errorf("Failed to create client for %s. err: %w", vuls2Conf.Repository, err)
+	}
+	if repo.Reference.Reference == "" {
+		return false, xerrors.Errorf("unexpected repository format. expected: %q, actual: %q", []string{"<repository>@<digest>", "<repository>:<tag>", "<repository>:<tag>@<digest>"}, vuls2Conf.Repository)
+	}
+
+	desc, err := repo.Resolve(ctx, repo.Reference.Reference)
+	if err != nil {
+		return false, xerrors.Errorf("Failed to resolve %s. err: %w", vuls2Conf.Repository, err)
+	}
+
+	return desc.Digest.String() != *metadata.Digest, nil
+}
+
 func shouldDownload(vuls2Conf config.Vuls2Conf, now time.Time) (bool, error) {
 	if _, err := os.Stat(vuls2Conf.Path); err != nil {
 		if errors.Is(err, os.ErrNotExist) {

detector/vuls2/db_test.go

@@ -1,6 +1,7 @@
 package vuls2_test
 
 import (
+	"context"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -184,3 +185,62 @@ func schemaVersionBoltDB(t *testing.T) uint {
 	}
 	return sv
 }
+
+func Test_hasNewerRemote(t *testing.T) {
+	tests := []struct {
+		name       string
+		digest     *string
+		writeDB    bool
+		repository string
+		want       bool
+		wantErr    bool
+	}{
+		{
+			// A db that was built locally rather than fetched has no digest to
+			// compare, so the repository's db counts as newer and this answers
+			// without reaching the registry.
+			name:       "no digest recorded",
+			writeDB:    true,
+			repository: "ghcr.io/vulsio/vuls-nightly-db:nightly",
+			want:       true,
+		},
+		{
+			name:       "no db file",
+			repository: "ghcr.io/vulsio/vuls-nightly-db:nightly",
+			wantErr:    true,
+		},
+		{
+			name:       "unparsable repository",
+			digest:     func() *string { s := "sha256:a"; return &s }(),
+			writeDB:    true,
+			repository: "not a repository",
+			wantErr:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conf := config.Vuls2Conf{
+				Path:       filepath.Join(t.TempDir(), "vuls.db"),
+				Repository: tt.repository,
+			}
+			if tt.writeDB {
+				if err := putMetadata(types.Metadata{
+					LastModified:  *parse("2024-01-02T00:00:00Z"),
+					Downloaded:    parse("2024-01-02T00:00:00Z"),
+					SchemaVersion: schemaVersionBoltDB(t),
+					Digest:        tt.digest,
+				}, conf.Path); err != nil {
+					t.Fatalf("putMetadata() err = %v", err)
+				}
+			}
+
+			got, err := vuls2.HasNewerRemote(context.Background(), conf)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("hasNewerRemote() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Errorf("hasNewerRemote() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

detector/vuls2/export_test.go

@@ -1,7 +1,18 @@
 package vuls2
 
+import "github.com/MaineK00n/vuls2/pkg/db/session"
+
+// Open exposes Session.open so tests can force the lazy open and inspect what
+// it produced.
+func (s *Session) Open() (*session.Session, error) { return s.open() }
+
+// SkipsUpdate reports whether this Session is barred from downloading a db, so
+// a test can assert that the request path cannot fetch.
+func (s *Session) SkipsUpdate() bool { return s.vuls2Conf.SkipUpdate }
+
 var (
 	ShouldDownload = shouldDownload
+	HasNewerRemote = hasNewerRemote
 
 	PreConvertPkgs   = preConvertPkgs
 	PreConvertCPEs   = preConvertCPEs